seaweedfs

mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-12-26 10:47:54 +08:00

Author	SHA1	Message	Date
chrislu	3a58b7bac6	a little safer	2022-02-26 03:23:06 -08:00
chrislu	708e14fcfa	avoid possible too big memory allocation	2022-02-26 03:22:41 -08:00
chrislu	3345a50d9b	prefetch 2 chunks	2022-02-26 03:06:17 -08:00
chrislu	86ce69f709	remove logs	2022-02-26 03:00:20 -08:00
chrislu	b2a148cb4c	use file size as max range	2022-02-26 03:00:08 -08:00
chrislu	2ab0ad24a3	use memory pool	2022-02-26 02:59:19 -08:00
chrislu	28b395bef4	better control for reader caching	2022-02-26 02:16:47 -08:00
chrislu	3ad5fa6f6f	chunk cache adds function ReadChunkAt	2022-02-25 21:55:04 -08:00
chrislu	fc7a4957ea	fix mount2 options	2022-02-25 21:22:44 -08:00
chrislu	72c0233938	less logs	2022-02-25 15:34:24 -08:00
chrislu	101e6d80b0	mount2: listXattr return ok if xattr is empty	2022-02-25 14:38:56 -08:00
chrislu	03466f955e	rename: delete source entry metadata only, skipping hard links	2022-02-25 02:57:54 -08:00
chrislu	8080fe4cc1	logs	2022-02-25 02:56:23 -08:00
chrislu	e423548673	rename: pass along entry metadata	2022-02-25 02:53:37 -08:00
chrislu	e8110bb54c	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-02-25 01:17:28 -08:00
chrislu	202a29d014	refactoring	2022-02-25 01:17:26 -08:00
chrislu	91d6785cf3	define metadata action types	2022-02-25 00:54:16 -08:00
chrislu	be3fc77391	mount2: use consistent inode	2022-02-25 00:53:27 -08:00
Chris Lu	9873bae115	Merge pull request #2706 from guo-sj/fix_log_info	2022-02-24 22:48:19 -08:00
guosj	121b31f750	fix incorrect log information	2022-02-25 13:41:20 +08:00
chrislu	ceaf993a27	mount2: add rdev	2022-02-24 14:51:25 -08:00
chrislu	95717d1006	simpler output	2022-02-24 13:50:08 -08:00
chrislu	b05962b90e	rename: handle hard links	2022-02-24 02:59:00 -08:00
chrislu	e31ec04f4f	mount2: POSIX deleted opened file nlink should be 0	2022-02-24 01:59:37 -08:00
chrislu	419e355e9e	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-02-24 01:31:07 -08:00
chrislu	91f0481f4e	mount2: SetAttr set mode correctly	2022-02-24 01:31:04 -08:00
Chris Lu	97a4b66df7	Merge pull request #2704 from guo-sj/fix_bugs_in_return_value fix return value in storage/volume_vacuum.go:444	2022-02-24 00:49:29 -08:00
chrislu	abe7214c1f	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-02-24 00:37:59 -08:00
chrislu	b93d57da31	mount2: dir read opened file	2022-02-24 00:37:57 -08:00
guosj	3e7aa1caf5	fix return value in storage/volume_vacuum.go:444	2022-02-24 15:54:36 +08:00
Chris Lu	6834df77a0	Merge pull request #2701 from guo-sj/fix_bugs_in_return_value	2022-02-23 19:23:37 -08:00
guosj	26f3ab8d4b	fix a return bug in func (c *commandVacuum) Do	2022-02-24 09:04:38 +08:00
chrislu	320637dc7a	use "mv.from" for moving files	2022-02-23 15:34:42 -08:00
chrislu	c29bc9a367	fix error handling	2022-02-23 15:34:25 -08:00
Chris Lu	da58c748bc	Merge pull request #2698 from guo-sj/fix_bugs_in_return_value	2022-02-23 00:41:30 -08:00
Chris Lu	bd092d8318	Merge pull request #2697 from guo-sj/fix_bugs_in_return_value	2022-02-23 00:30:52 -08:00
guosj	d68c27f82d	fix another return value bug	2022-02-23 16:21:25 +08:00
guosj	8f9aa0cddd	fix bugs in return value	2022-02-23 16:17:48 +08:00
Tuan Vuong	d2ec62656d	initialize master address in iam options	2022-02-23 12:01:54 +07:00
banjiaojuhao	6ab09e9071	filer_http: support uploading file with offset	2022-02-22 00:15:00 +08:00
chrislu	497ebbbd45	2.90	2022-02-20 22:00:13 -08:00
chrislu	56da3494cb	leveldb3: adjust memory allocation for each bucket	2022-02-20 13:40:13 -08:00
banjiaojuhao	e6126cef62	filer_web: support moving entry	2022-02-20 23:56:23 +08:00
chrislu	7bc67399e4	listing for filer stores without prefixed query: break if no more progress	2022-02-19 10:12:51 -08:00
chrislu	248c0c8087	minor	2022-02-19 09:54:09 -08:00
chrislu	4ee0a6f47b	filer store: reduce one possible listing operation	2022-02-19 00:43:42 -08:00
chrislu	b3594278c9	optimize a bit	2022-02-18 23:36:10 -08:00
chrislu	61811dc2f1	comments	2022-02-18 22:14:40 -08:00
chrislu	daa27b2119	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-02-18 20:55:10 -08:00
chrislu	9014d00fd0	Revert "s3: listObjectParts return ErrNoSuchUpload if does not exist" This reverts commit `6cf2e7d493`.	2022-02-18 20:54:54 -08:00
banjiaojuhao	4c30934cd9	filer: support get file entry	2022-02-18 22:52:26 +08:00
chrislu	63062ed7f0	mount2: fix unlink	2022-02-18 01:10:53 -08:00
chrislu	93e12d5f30	fix build	2022-02-18 00:48:00 -08:00
chrislu	d62370d4e0	add todo	2022-02-18 00:47:15 -08:00
chrislu	e8ce30fdc5	mount2: adjust file mode	2022-02-18 00:47:02 -08:00
chrislu	f9d33f70b0	return fuse.Status when looking up by inode	2022-02-18 00:45:43 -08:00
chrislu	b9cf4f12fc	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-02-16 21:33:51 -08:00
chrislu	49b84b6e2a	list entries while reading from remote	2022-02-16 21:32:15 -08:00
Chris Lu	38dcaaa76e	Merge pull request #2683 from guo-sj/fix_fsconfigure_bug fix minor bug in commandFsConfigure.Do	2022-02-16 18:16:47 -08:00
guosj	ca121ecd66	fix minor bug in commandFsConfigure.Do	2022-02-17 09:57:52 +08:00
chrislu	65a19e3abc	fix listing with correct inode	2022-02-16 17:01:39 -08:00
chrislu	6ac066d1dc	count lookup or not	2022-02-16 16:49:03 -08:00
chrislu	a129bda7d9	sync data first before stopping	2022-02-16 09:11:34 -08:00
chrislu	118d0e01a8	less logs	2022-02-16 08:45:07 -08:00
chrislu	a6bc67c34c	less logs	2022-02-16 08:38:51 -08:00
chrislu	2facd65998	fix second listing	2022-02-16 08:16:27 -08:00
chrislu	1013fc90d8	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-02-16 07:57:11 -08:00
chrislu	22739c653a	clean up	2022-02-16 07:57:08 -08:00
Chris Lu	3cb19cf6db	Merge pull request #2680 from kmlebedev/volume_healthz	2022-02-16 04:43:44 -08:00
Konstantin Lebedev	9ea09cc41c	healthz check to avoid drain pod with last replicas	2022-02-16 14:18:36 +05:00
chrislu	65bfeafb5a	add back mkdir	2022-02-16 01:09:21 -08:00
chrislu	37e8fce841	clean up cache on exit	2022-02-16 00:39:21 -08:00
chrislu	1560ec7e26	remove unused code	2022-02-16 00:37:24 -08:00
chrislu	3cbce878f2	mount2: fix directory pagination	2022-02-15 22:42:10 -08:00
chrislu	df51e0c042	mongodb: remove ErrNoDocuments checking since it is only for creating single record, not for querying	2022-02-15 16:18:36 -08:00
chrislu	98cce6a150	mongodb: ensure closing cursor	2022-02-15 16:15:44 -08:00
Chris Lu	aa7fc299b8	Merge pull request #2676 from banjiaojuhao/add_filer_store-redis_lua FilerStore: add redis_lua	2022-02-15 12:10:34 -08:00
banjiaojuhao	fc3b75f2f8	weed/operation/delete_content.go: nil pointer dereference	2022-02-15 23:07:03 +08:00
banjiaojuhao	b5ec346700	FilerStore: add redis_lua	2022-02-15 20:54:57 +08:00
chrislu	17ac5244c3	mount2: avoid double listing directories	2022-02-15 01:44:17 -08:00
guosj	db7766323e	complete project code, remain test code	2022-02-15 17:35:33 +08:00
chrislu	222798d926	mount2: fix for read dir plus on linux	2022-02-15 00:29:17 -08:00
chrislu	4e72863ba5	mount2 add debug mode	2022-02-15 00:26:30 -08:00
Chris Lu	64afbc5235	Merge pull request #2640 from Radtoo/fsck_with_delete Added basic deletion capabilities to fsck.	2022-02-14 23:00:01 -08:00
chrislu	4e181db21a	mount: default disable cache * Prevent cases as https://github.com/seaweedfs/seaweedfs-csi-driver/issues/43 * Improve read write benchmarks * Improve AI training performance. Most of the files are just read once.	2022-02-14 20:42:33 -08:00
chrislu	ebc22625b4	less noisy on mac	2022-02-14 16:38:01 -08:00
chrislu	377bf31445	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-02-14 13:48:51 -08:00
chrislu	e8420aaed7	fix building for windows freebsd	2022-02-14 13:48:48 -08:00
Chris Lu	d3ee621fce	Merge pull request #2661 from garenchan/ck-dev1	2022-02-14 10:08:37 -08:00
Konstantin Lebedev	526094d2da	StopTimeout 30 sec	2022-02-14 21:42:27 +05:00
Konstantin Lebedev	275e9a4e86	reduce to default http server KillTimeout and StopTimeout	2022-02-14 21:38:24 +05:00
garenchan	bd032eabe7	[UPDATE] Make heartbeat interval and election timeout of masters configurable.	2022-02-14 21:09:07 +08:00
chrislu	ff666104c4	fix GOOS	2022-02-14 03:14:05 -08:00
chrislu	05724a68d4	skip other OS	2022-02-14 02:59:51 -08:00
chrislu	4244ef8b72	fix building on linux	2022-02-14 02:59:27 -08:00
chrislu	b9c2bff931	clean up	2022-02-14 02:14:26 -08:00
chrislu	a990cd29cd	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-02-14 01:52:25 -08:00
chrislu	6a40fd1c65	2.89	2022-02-14 01:52:16 -08:00
Chris Lu	305418fac5	Merge pull request #2667 from kmlebedev/syncReplicatedWrite atomically write to replicas	2022-02-14 01:48:21 -08:00
Chris Lu	aa13168b4d	Merge pull request #2668 from chrislusf/mount2 Mount2	2022-02-14 01:41:09 -08:00
chrislu	fe57a2e770	file set attribute	2022-02-14 01:36:10 -08:00
Konstantin Lebedev	0ed76a0556	clearly	2022-02-14 14:10:06 +05:00
chrislu	dbeeda8123	listen for metadata updates	2022-02-14 01:09:31 -08:00
Konstantin Lebedev	36013f63ed	https://github.com/chrislusf/seaweedfs/issues/2648	2022-02-14 13:59:12 +05:00
chrislu	7286e525ad	support write	2022-02-13 23:27:11 -08:00
chrislu	2b955c1713	support read	2022-02-13 22:50:44 -08:00
chrislu	f3c1e00521	rename	2022-02-13 19:16:56 -08:00
chrislu	bb9919b07a	add open release, refactor	2022-02-13 19:14:34 -08:00
Eng Zer Jun	b92df1654c	test: use `T.TempDir` to create temporary test directory The directory created by `T.TempDir` is automatically removed when the test and all its subtests complete. Reference: https://pkg.go.dev/testing#T.TempDir Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>	2022-02-14 10:38:13 +08:00
chrislu	072f923a9b	add doc	2022-02-13 16:56:35 -08:00
chrislu	3d0e9e5197	supports renaming	2022-02-13 16:34:57 -08:00
chrislu	417b59b893	doc	2022-02-13 14:37:47 -08:00
chrislu	24290fed9d	add comments	2022-02-13 06:02:21 -08:00
chrislu	6200b6abb1	avoid fatal message in some edge cases	2022-02-13 05:59:10 -08:00
chrislu	6a921e15f3	forget() factor in nlookup	2022-02-13 05:49:29 -08:00
chrislu	f8af0f93d9	support link	2022-02-13 05:29:43 -08:00
chrislu	0381338d40	fix import	2022-02-13 04:23:06 -08:00
chrislu	a1ef0e48a9	doc	2022-02-13 04:22:02 -08:00
chrislu	be40ff6048	add symlink	2022-02-13 03:50:16 -08:00
chrislu	3d93570979	supports forget	2022-02-13 03:31:47 -08:00
chrislu	6a42cb6b0b	supports mknod, unlink	2022-02-13 03:09:24 -08:00
chrislu	813b868b9a	add rmdir	2022-02-13 01:43:11 -08:00
chrislu	e85ca10a1a	add mkdir	2022-02-13 01:34:19 -08:00
chrislu	21046c6a28	split files	2022-02-13 01:05:30 -08:00
chrislu	a4c9223b9d	support xattr	2022-02-13 00:58:46 -08:00
chrislu	c81833a192	add directory setAttr	2022-02-12 23:08:56 -08:00
chrislu	5c48c23235	remove println	2022-02-12 22:45:07 -08:00
chrislu	7cfbf1e85f	fix ok status	2022-02-12 22:41:45 -08:00
chrislu	4c75fd5f9c	sync format	2022-02-12 22:41:29 -08:00
chrislu	b0a5193e32	working	2022-02-12 22:21:30 -08:00
chrislu	661a34e23d	Merge branch 'master' into mount2	2022-02-12 17:58:32 -08:00
chrislu	5e814afe88	blocks count	2022-02-12 17:58:12 -08:00
Radtoo	724ce04b1c	We picked the second flag's name.	2022-02-13 00:53:35 +01:00
chrislu	5a0a709016	it runs, but directory listing output is not showing up	2022-02-12 05:27:16 -08:00
chrislu	866981d8ac	rename	2022-02-12 02:49:15 -08:00
chrislu	72faae91e1	implement read directory and read directory plus	2022-02-12 02:48:44 -08:00
chrislu	a10c28ba82	simplify	2022-02-12 01:59:36 -08:00
chrislu	f4d88862c4	can attr root directory	2022-02-12 01:54:16 -08:00
chrislu	180445f5a8	change to use fuse file system	2022-02-11 21:35:09 -08:00
chrislu	45a0fda9bd	need to follow https://github.com/hanwen/go-fuse/blob/master/fuse/api.go	2022-02-11 03:12:52 -08:00
chrislu	f87da798a4	to be re-written following fuse virtual file system	2022-02-11 03:09:30 -08:00
chrislu	9a913457dd	supports stats	2022-02-10 23:23:47 -08:00
chrislu	b6143de52a	mount with name	2022-02-10 22:43:55 -08:00
chrislu	7a0c35674c	clean up previously mounted folder	2022-02-10 20:46:53 -08:00
chrislu	c3f9d9fa2e	initial setup	2022-02-10 20:32:13 -08:00
Radtoo	fbe7ed7927	Revert the the flag to "reallyDeleteFromVolume".	2022-02-10 20:01:48 +01:00
chrislu	21aaa4c1f1	ec.encode: calculate free ec slots based on (maxVolumeCount-volumeCount) fix https://github.com/chrislusf/seaweedfs/issues/2642	2022-02-08 01:51:13 -08:00
chrislu	13f6ec1c4e	test checking ec distribution	2022-02-08 01:50:05 -08:00
chrislu	f34c2ff7c5	use embed txt	2022-02-08 01:14:27 -08:00
chrislu	c8c7c10c3f	volume.tier.move: avoid double counting related to https://github.com/chrislusf/seaweedfs/issues/2637	2022-02-08 00:57:35 -08:00
chrislu	f18803424a	volume.balance: add delay during tight loop fix https://github.com/chrislusf/seaweedfs/issues/2637	2022-02-08 00:53:55 -08:00
chrislu	9860405974	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-02-07 23:08:57 -08:00
chrislu	85c1615b43	filer read empty file may cause OOM in some cases fix https://github.com/chrislusf/seaweedfs/issues/2641	2022-02-07 23:08:54 -08:00
Chris Lu	a2ac540ecc	Merge pull request #2645 from guol-fnst/fix_Precedence fix preconditions	2022-02-07 18:32:16 -08:00
root	7f0c793083	fix preconditions according to https://tools.ietf.org/id/draft-ietf-httpbis-p4-conditional-26.html#preconditions	2022-02-08 10:13:19 +08:00
chrislu	b1cff07ab0	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-02-07 11:38:01 -08:00
chrislu	81f86c381d	s3: avoid nil response fix https://github.com/chrislusf/seaweedfs/issues/2636	2022-02-07 11:37:59 -08:00
Chris Lu	f46763d74f	Merge pull request #2644 from kmlebedev/vacum_fsync updated needle with fsync	2022-02-07 11:33:58 -08:00
Konstantin Lebedev	ef541972f8	updated needle with fsync	2022-02-08 00:10:53 +05:00
Chris Lu	1b7fb3ce84	Merge pull request #2643 from kmlebedev/graceful_volume_stop	2022-02-07 09:58:26 -08:00
Konstantin Lebedev	fb97e234c9	skips compact if store is stopping	2022-02-07 20:16:15 +05:00
chrislu	9405eaefdb	filer.sync: fix replicating partially updated file Run two servers with volumes and fillers: server -dir=Server1alpha -master.port=11000 -filer -filer.port=11001 -volume.port=11002 server -dir=Server1sigma -master.port=11006 -filer -filer.port=11007 -volume.port=11008 Run Active-Passive filler.sync: filer.sync -a localhost:11007 -b localhost:11001 -isActivePassive Upload file to 11007 port: curl -F file=@/Desktop/9.xml "http://localhost:11007/testFacebook/" If we request a file on two servers now, everything will be correct, even if we add data to the file and upload it again: curl "http://localhost:11007/testFacebook/9.xml" EQUALS curl "http://localhost:11001/testFacebook/9.xml" However, if we change the already existing data in the file (for example, we change the first line in the file, reducing its length), then this file on the second server will not be valid and will not be equivalent to the first file Снимок экрана 2022-02-07 в 14 21 11 This problem occurs on line 202 in the filer_sink.go file. In particular, this is due to incorrect mapping of chunk names in the DoMinusChunks function. The names of deletedChunks do not match the chunks of existingEntry.Chunks, since the first chunks come from another server and have a different addressing (name) compared to the addressing on the server where the file is being overwritten. Deleted chunks are not actually deleted on the server to which the file is replicated.	2022-02-07 03:46:28 -08:00
Radtoo	fbb14e0ea8	Adding separate toggle to purge absent vols While this toggle is basically required to clean out entries for deleted volumes, having a separate description + toggling this separately seems like a good idea so people get a chance to check if their volumes are all mounted/connected as expected. Also renamed forcePurge to just purge.	2022-02-06 23:46:52 +01:00
Radtoo	6b17f45da2	Also delete paths for which a volume is entirely absent.	2022-02-06 23:22:04 +01:00
Radtoo	fa0cfdfb7f	Added basic deletion capabilities to fsck.	2022-02-06 17:05:58 +01:00
chrislu	433fde4b18	move error to a separate file This file contains metric names for all errors The naming convention is ErrorSomeThing = "error.some.thing"	2022-02-04 22:57:51 -08:00
Chris Lu	a23fcb9a7c	Merge pull request #2634 from kmlebedev/errorMetrics error metrics for filer and store	2022-02-04 22:35:13 -08:00
Chris Lu	247bbabda5	Merge pull request #2632 from lapshin-vitaly/s3api_errors add s3api error for copy in file, not directory	2022-02-04 22:34:34 -08:00
chrislu	ced3b89395	add util package	2022-02-04 21:34:58 -08:00
chrislu	affe3c2c12	change to util.WriteFile	2022-02-04 21:32:27 -08:00
chrislu	76e297d64f	sync call to write file, avoid vif loading error fix https://github.com/chrislusf/seaweedfs/issues/2633	2022-02-04 11:14:04 -08:00
zerospiel	f3364fec99	weed/s3api: rearrange s3 methods handlers to ensure correct methods requesting Otherwise current calls for some methods (i.e. GetObjectAcl) ends up with wrong method selection (i.e. GetObject). Added generic comment rule of traversing methods	2022-02-04 15:14:48 +03:00
Konstantin Lebedev	9978f54acf	fix metric names	2022-02-04 16:45:16 +05:00
Konstantin Lebedev	3f4e17aa24	error metrics for filer and store	2022-02-04 14:07:14 +05:00
Lapshinn Vitaly	6bdc274d4d	add s3api error for copy in file, not directory	2022-02-04 03:28:37 +03:00
chrislu	6cf2e7d493	s3: listObjectParts return ErrNoSuchUpload if does not exist ubuntu@prod-master-1:~$ aws --endpoint http://10.244.15.66:8333 s3api abort-multipart-upload --bucket prod-cache --key multipart-test --upload-id 5347f936-6adc-43de-8e5c-1fd137c3b2bc ubuntu@prod-master-1:~$ aws --endpoint http://10.244.15.66:8333 s3api list-parts --bucket prod-cache --key multipart-test --upload-id 5347f936-6adc-43de-8e5c-1fd137c3b2bc { "Initiator": null, "Owner": null, "StorageClass": "STANDARD" } If we abort a multipart upload, it appears that records are left behind. We should get a 404 NoSuchKey error.	2022-02-03 12:34:16 -08:00
zerospiel	b54a65ba5a	weed/s3api: added new bucket handlers for more compatibility with AWS S3 Protocol Otherwise any requests to the underlying handlers results in calls to ListObjects (v1) that may intensively load gateway and volume servers. Added the following handlers with default responses: - GetBucketLocation - GetBucketRequestPayment Added the following handlers with NotFound and NotImplemented responses: - PutBucketAcl - GetBucketPolicy - PutBucketPolicy - DeleteBucketPolicy - GetBucketCors - PutBucketCors - DeleteBucketCors	2022-02-03 17:17:05 +03:00
chrislu	6bee1e9714	[volume.check.disk] fix wrong logic to compare fix https://github.com/chrislusf/seaweedfs/issues/2626 minuend - subtrahend	2022-02-01 07:48:28 -08:00
chrislu	7270067289	2.88	2022-01-30 20:25:26 -08:00
chrislu	84c9bc4389	edge case: old entry was not replicated to remote storage	2022-01-30 20:23:24 -08:00
chrislu	b8490fe427	adjust volume count even when not applying the changes	2022-01-28 19:11:46 -08:00
Chris Lu	b3f0f170b4	Merge pull request #2618 from divanikus/master	2022-01-28 03:34:52 -08:00
chrislu	b9b684194f	remove max connection age following https://github.com/grpc/grpc-go/issues/3170#issuecomment-552517779	2022-01-27 02:28:22 -08:00
divanikus	67e3fe996a	async volumeTierMove	2022-01-26 18:22:31 +03:00
chrislu	62d815d1ca	use limited in memory buffer instead of swap file	2022-01-26 00:36:19 -08:00
Chris Lu	9596fce562	Merge pull request #2613 from kmlebedev/masterPromMetrics Master prom metrics	2022-01-25 08:13:14 -08:00
Konstantin Lebedev	5c9259fa3c	fix metrics master name	2022-01-25 14:42:47 +05:00
chrislu	7328962009	revert "POSIX: should not delete if a directory is not empty" revert `0c75f15062` Reported: Hi, about commit: `0c75f15062` POSIX: should not delete if a directory is not empty It should still delete with a command like rm -rf "${path}"/ because it is a forced delete, but now it gets fail to delete non-empty folder: [...] Can you enable the delete if it is forced?	2022-01-24 14:21:50 -08:00
Konstantin Lebedev	c9952759c4	metrics master is leader	2022-01-24 20:13:07 +05:00
Konstantin Lebedev	28efe31524	new master metrics	2022-01-24 19:09:43 +05:00
chrislu	3bba2124ef	use a sliding window of in-memory writable chunks	2022-01-23 23:02:05 -08:00
chrislu	520591e6ea	reset swap file chunk after uploading	2022-01-23 18:30:53 -08:00
chrislu	e185d90d24	2.87	2022-01-23 16:18:55 -08:00
chrislu	4a311c7f5e	dedup local metadata subscribers fix https://github.com/chrislusf/seaweedfs/discussions/2542	2022-01-23 16:14:22 -08:00
chrislu	bb6854b972	adjust log	2022-01-22 12:35:09 -08:00
chrislu	2bdd737971	rename file	2022-01-22 08:28:35 -08:00
chrislu	643bbbeb49	rename	2022-01-22 08:27:40 -08:00
chrislu	18ed06b420	use memory when under 16 chunks	2022-01-22 08:11:01 -08:00
chrislu	8e80f3cd65	move upload pipeline locking to a different file	2022-01-22 08:09:55 -08:00
chrislu	dde34fa99d	rename	2022-01-22 08:08:01 -08:00
chrislu	a2aa542370	rename	2022-01-22 08:07:11 -08:00
chrislu	c376ccc5a5	swap file based random write large file upload	2022-01-22 08:06:53 -08:00
chrislu	b9ae16fbc5	fix memory allocation	2022-01-22 08:05:04 -08:00
chrislu	1ee828b768	refactor do not expose internal offset	2022-01-22 06:34:29 -08:00
chrislu	02d0c12cdd	rename	2022-01-22 06:00:10 -08:00
chrislu	5dea5c0449	refactor	2022-01-22 05:59:07 -08:00
chrislu	8aa6bf0bb9	refactoring	2022-01-22 05:40:10 -08:00
chrislu	d97bd54e63	just refactoring	2022-01-22 04:18:54 -08:00
chrislu	482014f9da	rename file	2022-01-22 03:50:18 -08:00
chrislu	3b4a9addaf	rename	2022-01-22 01:46:10 -08:00
chrislu	4acfc098e9	re-order	2022-01-22 01:43:14 -08:00
chrislu	e71dcfb3a6	add logging for memory allocation	2022-01-22 01:35:12 -08:00
chrislu	9d0f58c329	skip printing fs configuration	2022-01-21 13:29:47 -08:00
chrislu	ce2049cdb6	refactoring, move genFn before saveFn	2022-01-21 12:08:58 -08:00
chrislu	e47f63d159	enforce bucket quota	2022-01-21 02:34:42 -08:00
chrislu	6e57d8d0de	s3: check bucket usage and adjust read only according to quota	2022-01-21 02:15:27 -08:00
chrislu	606667f205	able to configure the quota for a bucket	2022-01-21 01:42:20 -08:00
chrislu	b1063162b6	display bucket quota	2022-01-21 00:55:04 -08:00
chrislu	f103491912	s3: list bucket size from weed shell	2022-01-21 00:26:49 -08:00
chrislu	6c7135d77e	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-01-21 00:17:06 -08:00
chrislu	ea57654e34	refactoring	2022-01-21 00:16:50 -08:00
Chris Lu	7c66f3b5fb	Merge pull request #2602 from kmlebedev/master_metrics master metricsHttpPort	2022-01-20 09:26:25 -08:00
chrislu	b3e526ba95	url should be always using forward slash	2022-01-19 22:16:26 -08:00
Konstantin Lebedev	77c98b657e	master metricsHttpPort	2022-01-19 21:43:22 +05:00
chrislu	77362700e1	S3: fail fast when "X-Amz-Copy-Source" is a folder fix #2593	2022-01-18 12:04:40 -08:00
chrislu	05c3c3f56b	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-01-17 23:38:13 -08:00
chrislu	9b77f0054e	2.86	2022-01-17 23:38:03 -08:00
Chris Lu	ec254d8a89	Merge pull request #2597 from guol-fnst/gocql_to add gocql timeout setting	2022-01-17 23:35:13 -08:00
chrislu	9274557552	keep dirty pages based on temp file	2022-01-17 23:23:49 -08:00
guol-fnst	da9540e666	add gocql timeout setting	2022-01-18 15:21:13 +08:00
chrislu	c87b8f4c30	S3: fail fast when "X-Amz-Copy-Source" is a folder fix https://github.com/chrislusf/seaweedfs/issues/2593	2022-01-17 23:09:37 -08:00
chrislu	b2acfd75e9	ensure entry view cache is invalidated	2022-01-17 23:02:30 -08:00
chrislu	f4ad63528a	wait for reading threads to complete before dropping sealed chunks	2022-01-17 22:24:44 -08:00
chrislu	0a3f95ca01	more logs	2022-01-17 20:41:00 -08:00
chrislu	b068bc291d	testing with always resetting entry view cache	2022-01-17 20:07:01 -08:00
chrislu	047446d5ca	remove extra async execution	2022-01-17 15:50:11 -08:00
chrislu	7bf7af971b	more logs	2022-01-17 14:15:10 -08:00
chrislu	fc22071a2f	more logs	2022-01-17 14:02:37 -08:00
chrislu	381f4e73a0	delete actual reference first	2022-01-17 13:56:47 -08:00
chrislu	0ba88596e8	invalidate filehandle entry view cache	2022-01-17 13:53:30 -08:00
chrislu	1734017ba1	add test	2022-01-17 13:40:41 -08:00
chrislu	da7f13e73e	Revert "testing skip memory management" This reverts commit `6c908352cb`.	2022-01-17 03:21:31 -08:00
chrislu	6c908352cb	testing skip memory management	2022-01-17 03:19:24 -08:00
chrislu	77d9993f38	remove unused variables	2022-01-17 03:19:11 -08:00
chrislu	f710d5ffca	a little speed up	2022-01-17 03:19:00 -08:00
chrislu	fc0628c038	working	2022-01-17 01:53:56 -08:00
chrislu	1bd6d289d4	better locking on file handle	2022-01-15 05:45:29 -08:00
chrislu	2bfeb5d1c8	add filer to iam option	2022-01-15 03:37:52 -08:00
chrislu	b17c426e99	weed server: optionally start IAM service related to https://github.com/chrislusf/seaweedfs/issues/2560	2022-01-13 22:49:49 -08:00
chrislu	3c8b74318e	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-01-13 13:03:07 -08:00
chrislu	8907e6a40a	add more help messages	2022-01-13 13:03:04 -08:00
banjiaojuhao	45e9c83421	padding zero for sparse file	2022-01-13 22:21:22 +08:00
chrislu	fe5b9e39cc	POSIX: check permission when removing items	2022-01-13 02:07:39 -08:00
chrislu	1453263b63	remove dead code	2022-01-13 02:02:04 -08:00
chrislu	e69c374956	minor	2022-01-13 02:01:53 -08:00
chrislu	9b954dc0d4	adjust make file	2022-01-13 01:33:13 -08:00
chrislu	f2847f1266	POSIX: check deletion permission	2022-01-12 23:58:11 -08:00
chrislu	0c75f15062	POSIX: should not delete if a directory is not empty	2022-01-12 23:57:54 -08:00
chrislu	de27058d0b	POSIX: differentiate device and char device	2022-01-12 21:45:38 -08:00
chrislu	d400a11832	POSIX: adjust source file ctime SeaweedFS uses mtime as ctime	2022-01-12 21:45:18 -08:00
chrislu	b44f05a2d0	POSIX: change timestamp on each attribute change	2022-01-12 19:31:25 -08:00
chrislu	15c01d8b7f	add some notes	2022-01-12 15:04:48 -08:00
chrislu	107a4884a8	shell: tighter memory allocation	2022-01-12 14:59:29 -08:00
chrislu	fec8428fd8	POSIX: different inode for same named different file types	2022-01-12 11:51:13 -08:00
chrislu	e82ad60122	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-01-12 11:07:49 -08:00
chrislu	caf0a3486b	POSIX: adjust ctime for file truncate	2022-01-12 11:07:39 -08:00
Konstantin Lebedev	edb753ab4d	https://github.com/chrislusf/seaweedfs/issues/2583	2022-01-12 16:04:59 +05:00
chrislu	adfd54e7c4	fix compilation	2022-01-12 01:24:24 -08:00
chrislu	6cc92817dc	add logs for request mode	2022-01-12 01:13:19 -08:00
chrislu	826a7b307e	master: remove hard coded filer settings in master.toml fix https://github.com/chrislusf/seaweedfs/issues/2529	2022-01-12 01:11:25 -08:00
chrislu	cd1ad88f30	POSIX: check name is too long ENAMETOOLONG	2022-01-12 00:16:00 -08:00
chrislu	2dcb8cb93b	POSIX: ensure file and directory inodes are different this is just an in memory representation. POSIX wants different inode numbers for the same named file or directory.	2022-01-11 23:44:48 -08:00
chrislu	5bb37d5905	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-01-11 23:29:12 -08:00
chrislu	10ecf80ca1	add a debug capability to list all metadata keys	2022-01-11 23:25:04 -08:00
Kyle Sanderson	9e012001be	filer.copy: don't crash when volume creation fails panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 addr=0x28 pc=0x1d58247] goroutine 7482 [running]: github.com/chrislusf/seaweedfs/weed/command.(FileCopyWorker).uploadFileInChunks.func1(0x2) /go/src/github.com/chrislusf/seaweedfs/weed/command/filer_copy.go:488 +0x2a7 created by github.com/chrislusf/seaweedfs/weed/command.(FileCopyWorker).uploadFileInChunks /go/src/github.com/chrislusf/seaweedfs/weed/command/filer_copy.go:455 +0x225 panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 addr=0x28 pc=0x1d58247] goroutine 7480 [running]: github.com/chrislusf/seaweedfs/weed/command.(FileCopyWorker).uploadFileInChunks.func1(0x0) /go/src/github.com/chrislusf/seaweedfs/weed/command/filer_copy.go:488 +0x2a7 created by github.com/chrislusf/seaweedfs/weed/command.(FileCopyWorker).uploadFileInChunks /go/src/github.com/chrislusf/seaweedfs/weed/command/filer_copy.go:455 +0x225	2022-01-11 22:22:39 -08:00
chrislu	1a7d5b5b5e	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2022-01-11 12:24:56 -08:00
chrislu	41daecfdca	Update mount_std.go	2022-01-11 12:23:12 -08:00
chrislu	2d0ccc4d34	add logs	2022-01-11 12:23:01 -08:00
Chris Lu	abe5da7d2c	Merge pull request #2575 from Radtoo/fix_paths2 Fix paths2	2022-01-11 12:04:30 -08:00
chrislu	b8fbf19e9a	mount: rename follow POSIX	2022-01-11 03:23:03 -08:00
chrislu	6a12520a96	fix logging	2022-01-10 01:00:11 -08:00
chrislu	cbc055dc2b	mount: file fsync fix https://github.com/chrislusf/seaweedfs/issues/2561	2022-01-10 00:52:16 -08:00
chrislu	19555385f7	2.85	2022-01-09 19:30:23 -08:00
Radtoo	389002f195	Using positional arguments rather than option flag to enable better shell usage	2022-01-08 16:52:12 +01:00
Radtoo	fba1efb77a	Now works with a single file too Parsing removed from doFixOneVolume Needle init removed from runFix	2022-01-08 16:31:53 +01:00
chrislu	110d5a5233	support fixing a collection of volumes, or volumes under one directory	2022-01-07 14:52:16 -08:00
chrislu	60dc450091	skip fixing read only volumes fix https://github.com/chrislusf/seaweedfs/issues/2562	2022-01-06 09:52:28 -08:00
chrislu	3df8f96117	avoid changing inode	2022-01-06 01:36:11 -08:00
chrislu	67b0645808	mount: need to change entry name after renaming	2022-01-05 21:27:41 -08:00
chrislu	4de060daa6	mount: skip special character in the filenames fix https://github.com/chrislusf/seaweedfs/issues/2559	2022-01-05 03:57:24 -08:00
chrislu	e76105e2ab	fix auth permission checking	2022-01-03 21:05:20 -08:00
chrislu	a7887166cf	wildcard prefix to restrict access to directories in s3 bucket https://github.com/chrislusf/seaweedfs/discussions/2551	2022-01-03 15:39:36 -08:00
chrislu	5799a20f71	2.84	2022-01-02 17:05:19 -08:00
Chris Lu	42c849e0df	Merge branch 'master' into metadata_follow_with_client_id	2022-01-02 01:07:30 -08:00
Chris Lu	9b94177380	Merge pull request #2543 from skurfuerst/seaweedfs-158 FEATURE: add JWT to HTTP endpoints of Filer and use them in S3 Client	2022-01-01 22:34:13 -08:00
Sebastian Kurfuerst	c35660175d	BUGFIX: ensure Authorization header is only added once	2021-12-31 22:06:18 +01:00
Sebastian Kurfuerst	1cd3b6b4e1	BUGFIX: security.toml contained wrong keys	2021-12-31 22:05:41 +01:00
Sebastian Kurfuerst	10404c4275	FEATURE: add JWT to HTTP endpoints of Filer and use them in S3 Client - one JWT for reading and one for writing, analogous to how the JWT between Master and Volume Server works - I did not implement IP `whiteList` parameter on the filer Additionally, because http_util.DownloadFile now sets the JWT, the `download` command should now work when `jwt.signing.read` is configured. By looking at the code, I think this case did not work before. ## Docs to be adjusted after a release Page `Amazon-S3-API`: ``` # Authentication with Filer You can use mTLS for the gRPC connection between S3-API-Proxy and the filer, as explained in [Security-Configuration](Security-Configuration) - controlled by the `grpc.` configuration in `security.toml`. Starting with version XX, it is also possible to authenticate the HTTP operations between the S3-API-Proxy and the Filer (especially uploading new files). This is configured by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. With both configurations (gRPC and JWT), it is possible to have Filer and S3 communicate in fully authenticated fashion; so Filer will reject any unauthenticated communication. ``` Page `Security Overview`: ``` The following items are not covered, yet: - master server http REST services Starting with version XX, the Filer HTTP REST services can be secured with a JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. ... Before version XX: "weed filer -disableHttp", disable http operations, only gRPC operations are allowed. This works with "weed mount" by FUSE. It does not work* with the [S3 Gateway](Amazon S3 API), as this does HTTP calls to the Filer. Starting with version XX: secured by JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. This now works with the [S3 Gateway](Amazon S3 API). ... # Securing Filer HTTP with JWT To enable JWT-based access control for the Filer, 1. generate `security.toml` file by `weed scaffold -config=security` 2. set `filer_jwt.signing.key` to a secret string - and optionally filer_jwt.signing.read.key` as well to a secret string 3. copy the same `security.toml` file to the filers and all S3 proxies. If `filer_jwt.signing.key` is configured: When sending upload/update/delete HTTP operations to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.key`. If `filer_jwt.signing.read.key` is configured: When sending GET or HEAD requests to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.read.key`. The S3 API Gateway reads the above JWT keys and sends authenticated HTTP requests to the filer. ``` Page `Security Configuration`: ``` (update scaffold file) ... [filer_jwt.signing] key = "blahblahblahblah" [filer_jwt.signing.read] key = "blahblahblahblah" ``` Resolves: #158	2021-12-30 14:45:27 +01:00
chrislu	34742be029	remove duplicated metadata subscription in filer https://github.com/chrislusf/seaweedfs/issues/2545	2021-12-30 01:51:52 -08:00
chrislu	5c87fcc6d2	add client id for all metadata listening clients	2021-12-30 00:23:57 -08:00
chrislu	fb434318e3	dynamically adjust connection timeout better fix for https://github.com/chrislusf/seaweedfs/issues/2541	2021-12-29 22:44:39 -08:00
chrislu	5788bf2270	s3: increase timeout limit https://github.com/chrislusf/seaweedfs/issues/2541	2021-12-29 22:21:02 -08:00
Sebastian Kurfuerst	fcc09cef6f	Refactor: pass in claim type into security.DecodeJwt	2021-12-29 12:40:41 +01:00
Sebastian Kurfuerst	d156d410ef	rename security.GenJwt to security.GenJwtForVolumeServer	2021-12-29 12:39:41 +01:00
Sebastian Kurfuerst	eda4c43a08	fix typo in error message	2021-12-29 12:38:14 +01:00
chrislu	498661e3bb	mount: remove limits on number of parallel requests	2021-12-28 17:41:01 -08:00
chrislu	9a00c17555	reader: avoid wrong pattern detection due to lock waiting	2021-12-28 16:30:33 -08:00
chrislu	f7a6f6b4c0	if this is enabled, there are some "bus error" with git clone	2021-12-28 13:44:52 -08:00
chrislu	0da2dfd640	fuse: change to direct io mode before and after: chrislu$ time dd if=/dev/random of=/Users/chrislu/tmp/mm/testfile bs=131072 count=8192 8192+0 records in 8192+0 records out 1073741824 bytes transferred in 4.534068 secs (236816430 bytes/sec) dd if=/dev/random of=/Users/chrislu/tmp/mm/testfile bs=131072 count=8192 0.01s user 3.86s system 84% cpu 4.561 total chrislu$ time dd if=/dev/random of=/Users/chrislu/tmp/mm/testfile bs=131072 count=8192 8192+0 records in 8192+0 records out 1073741824 bytes transferred in 3.824072 secs (280784948 bytes/sec) dd if=/dev/random of=/Users/chrislu/tmp/mm/testfile bs=131072 count=8192 0.01s user 3.22s system 83% cpu 3.857 total	2021-12-28 12:22:56 -08:00
chrislu	3fd4da34a4	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2021-12-28 01:10:38 -08:00
chrislu	2422556456	monitor write pattern: avoid timing due to locking	2021-12-28 01:10:35 -08:00
chenkai	47c30e3add	filer list entries use context to break job	2021-12-28 15:03:41 +08:00
chrislu	80db8b13d8	bug: cleanup function was called twice	2021-12-27 20:53:02 -08:00
chrislu	67b723f74e	Filer Server API support fsync fix https://github.com/chrislusf/seaweedfs/issues/2528	2021-12-26 17:28:47 -08:00
chrislu	9f9ef1340c	use streaming mode for long poll grpc calls streaming mode would create separate grpc connections for each call. this is to ensure the long poll connections are properly closed.	2021-12-26 00:15:03 -08:00
chrislu	c935b9669e	2.83	2021-12-25 01:01:34 -08:00
chrislu	eb4ad2546f	use proper chunk size limit option	2021-12-24 22:52:18 -08:00
chrislu	41bbf320bb	use 2MB chunk size. cache size is the wrong option	2021-12-24 22:50:19 -08:00
chrislu	982ea85d81	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2021-12-24 22:40:07 -08:00
chrislu	083d8e9ece	add stream writer this should improve streaming write performance, which is common in many cases, e.g., copying large files. This is additional to improved random read write operations: `3e69d19380...19084d8791`	2021-12-24 22:38:22 -08:00
Sebastian Kurfürst	6db49100d6	BUGFIX: add access.ui setting to scaffolded security.toml ... The property is read here: `b70cb3e0b2/weed/server/volume_server.go (L69)`	2021-12-24 13:59:04 +01:00
chrislu	255a1c7dcd	refactor type names	2021-12-23 18:23:18 -08:00
chrislu	f77ca41769	refactor	2021-12-23 17:48:34 -08:00
chrislu	1d36884845	rename files	2021-12-23 17:47:58 -08:00
chrislu	2d1a1f5e03	rename variables and functions	2021-12-23 17:35:57 -08:00
chrislu	7bf48ee135	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2021-12-23 17:23:26 -08:00
chrislu	6de331b014	clean up	2021-12-23 17:23:21 -08:00
chrislu	032df784ed	chunked file works now	2021-12-23 17:17:32 -08:00
banjiaojuhao	083bf3a137	filer server: add "datacenter, rack and datanode" for path specific configuration	2021-12-23 23:25:05 +08:00
chrislu	c2aad1c7ff	detect non streaming mode on first write request	2021-12-22 17:20:44 -08:00
chrislu	b541e39a2c	fix tests	2021-12-22 16:17:30 -08:00
chrislu	2bc6fa90ff	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2021-12-22 16:05:43 -08:00
chrislu	0ec7bc6710	detect non streaming mode on the first read	2021-12-22 16:05:38 -08:00
chrislu	4c1368d621	fix test	2021-12-22 16:05:08 -08:00
Chris Lu	4e73705533	Merge pull request #2530 from banjiaojuhao/filer-upload-file-to-node filer server: allow upload file to specific dataNode	2021-12-22 12:49:15 -08:00
banjiaojuhao	08336be92e	filer server: allow upload file to specific dataNode	2021-12-22 21:57:26 +08:00
chrislu	7b78fc72b0	add page chunk interval list	2021-12-22 02:53:33 -08:00
chrislu	b7cd52636b	Merge branch 'master' of https://github.com/chrislusf/seaweedfs	2021-12-21 22:24:43 -08:00
chrislu	3981d65b68	remove println	2021-12-21 22:24:38 -08:00
chrislu	9a73319b45	mount: different write strategy for streaming write and random write	2021-12-21 17:28:55 -08:00
Chris Lu	b0665a15f4	Merge pull request #2527 from banjiaojuhao/master-assign-by-datanode	2021-12-21 08:56:51 -08:00
banjiaojuhao	dda6b90d25	assign fileId according to DataNode with empty DataCenter and Rack	2021-12-21 17:28:33 +08:00
chrislu	4b8dcff448	reverting default admin scripts fix https://github.com/chrislusf/seaweedfs/issues/2525 this new default value was introduced in 2.80 this affects production environments, e.g., EC is not desired, volume balancing is not preferred, etc.	2021-12-20 13:34:57 -08:00
chrislu	b21a67bbe6	add writer pattern object for later use	2021-12-20 11:53:48 -08:00
chrislu	4fd29dad86	remove writeOnly flag	2021-12-20 01:11:43 -08:00
chrislu	bc96682760	refactor, change file locations	2021-12-20 01:02:23 -08:00
chrislu	866c2657f0	avoid FUSE cache only for the first 512 bytes	2021-12-19 23:13:36 -08:00
chrislu	0cb9036f66	mount: only cache the first chunk on stream read	2021-12-19 23:06:03 -08:00
chrislu	a152f17937	mount: improve read performance on random reads	2021-12-19 22:43:14 -08:00
chrislu	85c526c583	s3: bind to a specific IP fix https://github.com/chrislusf/seaweedfs/issues/2516	2021-12-17 11:34:37 -08:00
chrislu	5eacff9d4f	log message adds server name address https://github.com/chrislusf/seaweedfs/issues/2514#issuecomment-995925733	2021-12-16 10:46:26 -08:00
chrislu	50ddd8c8e2	remove debug messages fix https://github.com/chrislusf/seaweedfs/issues/2514	2021-12-16 00:58:15 -08:00
chrislu	7210558c7b	s3: pass through s3 presigned headers fix https://github.com/chrislusf/seaweedfs/discussions/2502	2021-12-15 13:18:53 -08:00
chrislu	bf4d7affc0	gateway to remote object store: skip replicating multipart upload part files fix https://github.com/chrislusf/seaweedfs/issues/2509	2021-12-14 19:48:31 -08:00
chrislu	316f326464	add more help message	2021-12-13 13:14:36 -08:00
Konstantin Lebedev	969f513265	disable audit log for s3 statusHandler	2021-12-13 16:08:19 +05:00
Chris Lu	ab0c6a84bf	Merge pull request #2503 from kmlebedev/audit_log_nonblocking Audit log force async	2021-12-13 00:47:16 -08:00
Konstantin Lebedev	34779e8f38	force enable asynchronous I/O sending events to Fluentd	2021-12-13 13:39:39 +05:00
chrislu	c3b73ec23b	2.82	2021-12-12 23:25:24 -08:00
chrislu	7bf14452f7	s3: handle Response-Content-Disposition	2021-12-12 23:05:40 -08:00
chrislu	94e5c0a454	skip purging from ec volumes	2021-12-12 22:55:27 -08:00
chrislu	a2d3f89c7b	add lock messages	2021-12-10 13:24:38 -08:00
chrislu	991a3dca0d	rename file	2021-12-10 13:11:00 -08:00
kmlebedev	4f98553ba9	audit log SignatureVersion	2021-12-10 19:40:32 +05:00
chrislu	2d6fcdf83a	add delay if need to resume from disk	2021-12-09 13:13:05 -08:00
liubaojiang	c7abc34dbb	rename operation returns the correct old parent path	2021-12-10 00:02:57 +08:00
Konstantin Lebedev	98251fe16a	non blocking audit log	2021-12-09 19:47:16 +05:00
chrislu	40ee9b4d7b	sleep between the gap of each execution related to https://github.com/chrislusf/seaweedfs/issues/2476	2021-12-07 11:44:08 -08:00
Chris Lu	2ba08afed1	Merge pull request #2498 from kmlebedev/s3_audit_log	2021-12-07 09:35:48 -08:00
Konstantin Lebedev	10678cde81	audit log config	2021-12-07 18:20:52 +05:00
Konstantin Lebedev	4ec8715f20	audit log	2021-12-07 12:15:48 +05:00
Konstantin Lebedev	3ac48cd540	audit log	2021-12-07 12:13:21 +05:00
chrislu	7f0a97c7b6	shell: volume.list supports different verbosity level	2021-12-05 21:54:40 -08:00
chrislu	5ea9715721	2.81 also sync java client version to SeaweedFS version	2021-12-05 18:05:24 -08:00
chrislu	e6c026db65	volume.fix.replication: fix misplaced volumes fix https://github.com/chrislusf/seaweedfs/issues/2416	2021-12-05 16:56:25 -08:00
chrislu	53e2dee177	[volume.check.disk] was using the wrong source and target locations fix https://github.com/chrislusf/seaweedfs/issues/2268	2021-12-05 14:32:04 -08:00
chrislu	c146c76d10	avoid creating the same bucket with a different randomized name related to https://github.com/chrislusf/seaweedfs/issues/2492	2021-12-05 13:06:41 -08:00
chrislu	42d97a3442	adjust randomized bucket name	2021-12-05 12:36:58 -08:00
chrislu	488afa5002	volume: load volume can optionally be skipped, if ec volume exists fix https://github.com/chrislusf/seaweedfs/issues/2489	2021-12-05 02:28:52 -08:00
chrislu	59e58c4b23	volume: fix loading old volume format fix https://github.com/chrislusf/seaweedfs/issues/2487	2021-12-05 01:06:01 -08:00
chrislu	b70cb3e0b2	upgrade protoc to 3.17.3 $brew install protobuf $ protoc --version libprotoc 3.17.3 $ go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 $ go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1	2021-12-05 00:42:25 -08:00
chrislu	5c3b783310	s3: copy object API needs to escape special characters fix https://github.com/chrislusf/seaweedfs/issues/2482	2021-12-04 23:24:53 -08:00
Bl1tz23	83bffca7e6	fix MongodbStore.ListDirectoryEntries panic on find failure	2021-12-03 13:58:37 +03:00
Tanmoy Majumdar	ea09fb477a	return ' shouldRetry=true' so that filer can retry the failed chunk	2021-12-03 11:54:20 +06:00
Chris Lu	689f5513a9	redis3 supports sentinel	2021-11-29 01:09:51 -08:00
Chris Lu	7227cfddf5	2.80	2021-11-29 00:57:08 -08:00
Chris Lu	3d7390302d	add s3.clean.uploads -timeAgo=24h	2021-11-29 00:49:49 -08:00
Chris Lu	2f72c24498	skip the rest logic	2021-11-29 00:32:21 -08:00
Chris Lu	88ff8fc27b	ensure uploaded chunks are deleted on error	2021-11-29 00:28:26 -08:00
Chris Lu	bea26549f2	avoid empty extended attribute value	2021-11-28 23:50:52 -08:00
Chris Lu	ce2af0082e	revert	2021-11-28 23:35:22 -08:00
Chris Lu	1c9f3c7ac0	read deleted chunks when replcating data	2021-11-28 23:34:34 -08:00
Chris Lu	96c66ca2aa	read deleted chunks when replicating data	2021-11-28 23:33:03 -08:00
Chris Lu	f3a334965d	Update command_remote_uncache.go minor	2021-11-28 23:10:32 -08:00
Chris Lu	9ccfc1cfee	Update command_volume_fsck.go minor	2021-11-28 22:39:24 -08:00
Chris Lu	71a94267c9	Update command_volume_fsck.go remove unused code	2021-11-28 22:37:11 -08:00
Chris Lu	ad16221a35	adjust error log	2021-11-28 22:06:17 -08:00
Chris Lu	cf1586a34d	add logs for writing to remote file	2021-11-27 22:09:23 -08:00
Chris Lu	3a19eea97c	allocate memory by slabs	2021-11-27 12:13:00 -08:00
limd	8805c04128	fix redis2 sentinel config example	2021-11-25 19:20:02 +08:00
limd	ec03f22cc3	Merge remote-tracking branch 'origin/master'	2021-11-25 16:07:14 +08:00
limd	220797bd71	support redis sentinel	2021-11-25 15:57:03 +08:00
Chris Lu	6c27845be0	add retries when writing to remote s3 fix https://github.com/chrislusf/seaweedfs/issues/2465	2021-11-22 21:48:04 -08:00
Chris Lu	f3c789d662	2.79	2021-11-21 18:40:24 -08:00
Konstantin Lebedev	40abae7caa	Avoid forbiddening for HeadBucketHandler if owners are specified https://github.com/chrislusf/seaweedfs/issues/2434	2021-11-17 14:42:24 +05:00
Chris Lu	3d87aa767d	fix same dc and other dc	2021-11-16 09:14:01 -08:00
Chris Lu	a0ef6e3611	prevent nil response fix https://github.com/chrislusf/seaweedfs/issues/2452	2021-11-15 08:46:01 -08:00
Chris Lu	100c654ec3	2.78	2021-11-14 23:29:59 -08:00
Chris Lu	7bf891c00a	randomize same-dc servers and other-dc servers	2021-11-12 11:30:11 -08:00
Chris Lu	1f75f1f9dc	filer: fix mysql2 SQL template	2021-11-11 22:28:28 -08:00
Chris Lu	3abbaccb70	filer: fix mysql command to upsert	2021-11-11 22:27:13 -08:00
user	9668b15f38	Filtering by volume id is prioritized.	2021-11-11 18:18:56 +09:00
user	563a74a9eb	Volume filter by collection pattern added.	2021-11-11 18:01:47 +09:00
user	dbb8003ce3	Volume filter function added.	2021-11-11 17:45:17 +09:00
user	c387fe957b	Unused parameter removed.	2021-11-11 16:08:41 +09:00
Konstantin Lebedev	42f4aa824b	force overwrite s3-is-admin https://github.com/chrislusf/seaweedfs/issues/2433	2021-11-11 11:34:01 +05:00
Chris Lu	c77c0de418	make install	2021-11-09 12:34:57 -08:00
Chris Lu	a7eee8b14b	Merge pull request #2427 from Bl1tz23/master Fix: potencial iam identities data race	2021-11-09 08:53:48 -08:00
Chris Lu	1d4a61af5e	add brokers	2021-11-09 08:50:55 -08:00
Bl1tz23	c683409e92	s3: add RWMutex to iam, use RLock for concurrent reading	2021-11-09 18:11:06 +03:00
Bl1tz23	5018b22f36	s3: fix potencial iam identities data race	2021-11-09 12:19:50 +03:00
Chris Lu	59d1435d65	s3: avoid possible attacks by version	2021-11-08 17:47:56 -08:00
Chris Lu	5c99b8acc3	rename variables	2021-11-08 17:47:56 -08:00
Chris Lu	00fdab1284	adjust client name	2021-11-08 17:47:56 -08:00
Chris Lu	4729a57cc0	use constants	2021-11-08 17:47:56 -08:00
Chris Lu	d9dd72ea56	rename pacakge	2021-11-08 17:47:56 -08:00
Chris Lu	5cf332357b	2.77	2021-11-07 13:52:45 -08:00
Chris Lu	244841f080	adjust logs	2021-11-07 13:27:57 -08:00
Chris Lu	6e999f1176	s3: fix ListObject if more than 10000 objects	2021-11-07 12:39:36 -08:00
Chris Lu	a8b0f8864d	fix help message	2021-11-07 12:38:35 -08:00
Chris Lu	55973e8572	log unknown access key	2021-11-07 12:37:46 -08:00
Chris Lu	c4e22b5a9a	filer: deprecate "-peers" option	2021-11-06 14:36:45 -07:00
Chris Lu	751a7073e3	Merge branch 'flexible_filer_addition'	2021-11-06 14:26:26 -07:00
Chris Lu	e0fc2898e9	auto updated filer peer list	2021-11-06 14:23:35 -07:00
Chris Lu	04663c3611	remote.mount: print out metadata sync errors	2021-11-06 11:29:50 -07:00
Chris Lu	330d1fde7f	send peers info to filers	2021-11-06 11:29:50 -07:00
Chris Lu	84bb8e7365	send peers info to filers	2021-11-06 04:07:38 -07:00
Chris Lu	4b9c42996a	refactor grpc API	2021-11-05 18:11:40 -07:00
Chris Lu	5ea86ef1da	Revert "master: rename grpc function KeepConnected() to SubscribeVolumeLocationUpdates()" This reverts commit `af71ae11aa`.	2021-11-05 17:52:15 -07:00
Chris Lu	314c32514b	add command to list filers	2021-11-05 17:39:41 -07:00
Chris Lu	7348d2ae63	randomize traffic if replicated	2021-11-05 00:03:13 -07:00
Chris Lu	00ae965d8d	randomize a bit for ec shards distribution	2021-11-04 09:23:40 -07:00
Chris Lu	77f90ae288	add leader election in master	2021-11-04 00:54:38 -07:00
Chris Lu	35c37562bc	check auth only when enabled	2021-11-03 12:11:36 -07:00
Chris Lu	3c245c69d3	handle possible race condition	2021-11-03 01:36:39 -07:00
Chris Lu	af71ae11aa	master: rename grpc function KeepConnected() to SubscribeVolumeLocationUpdates()	2021-11-03 01:09:48 -07:00
Chris Lu	ab97b17e62	better printout	2021-11-02 23:45:47 -07:00
Chris Lu	e66865a8c5	adjust master client log level	2021-11-02 23:45:28 -07:00
Chris Lu	0c8dea9de8	go fmt	2021-11-02 23:39:16 -07:00
Chris Lu	5160eb08f7	shell: optionally read filer address from master	2021-11-02 23:38:45 -07:00
Chris Lu	18bfbf62fc	add one unit test	2021-11-02 23:36:43 -07:00
Chris Lu	e1ab8b01d0	s3: adjust permission for HEAD bucket operation fix https://github.com/chrislusf/seaweedfs/issues/2417#issuecomment-958391856	2021-11-02 16:28:55 -07:00
Chris Lu	b25661c6df	s3: skip permission checking for creating bucket if the bucket already exists fix https://github.com/chrislusf/seaweedfs/issues/2417 Rclone was trying to create the bucket even though the bucket already exists.	2021-11-02 13:13:36 -07:00
Chris Lu	1b90d60765	filer: paginate large directories to delete file chunks	2021-11-02 01:04:50 -07:00
Chris Lu	794375ca0a	adjust help message since both fullPercent and quietFor are needed.	2021-11-01 17:22:47 -07:00
Chris Lu	fc9e246592	2.76	2021-10-31 18:08:28 -07:00
Chris Lu	9cf756b188	S3: support CORS related to https://github.com/chrislusf/seaweedfs/issues/1271	2021-10-31 18:06:43 -07:00
Chris Lu	f17fa400d5	refactoring	2021-10-31 18:05:34 -07:00
Chris Lu	006b4ec0f8	refactoring	2021-10-31 18:02:08 -07:00
Chris Lu	edbf6d297b	filer.meta.tail: add example to send metadata to elastic search	2021-10-31 18:01:33 -07:00
Chris Lu	24858507cc	rename API to avoid confusion	2021-10-30 19:27:25 -07:00
Chris Lu	f5824f369c	avoid reporting error when it is a user request error	2021-10-29 14:14:29 -07:00
Chris Lu	c857cc7286	cloud tier: remove tagging since not all s3 vendors support this	2021-10-29 12:39:19 -07:00
Chris Lu	d04cdcf40d	s3 header add user agent	2021-10-29 12:28:24 -07:00
Konstantin Lebedev	dc3ae5e8a6	DeleteBucketLifecycleConfiguration return 204	2021-10-28 19:34:37 +05:00
Konstantin Lebedev	2afb5a13af	fix DeleteBucketLifecycleConfiguration	2021-10-28 18:30:33 +05:00
Chris Lu	2e76834e4d	filer store: redis2 fix wrong pagination	2021-10-27 23:46:25 -07:00
Chris Lu	54b6e0f3fd	adjust logs	2021-10-27 23:46:07 -07:00
Chris Lu	900e888695	filer: avoid possible prefixed pagination loop on unsupported filer stores	2021-10-27 23:45:48 -07:00
Chris Lu	d774fa6c9a	rename variable	2021-10-25 14:39:20 -07:00
Chris Lu	2539ba0b62	fix compilation	2021-10-25 14:38:48 -07:00
Chris Lu	5f2d7c1589	erasure coding: skip erasure coding if less than recommended 4 nodes	2021-10-25 14:38:11 -07:00
Chris Lu	c9d3fb4a30	2.75	2021-10-24 18:15:59 -07:00
Chris Lu	5435027ff0	volume copy: stream out copying progress and avoid grpc request timeout fix https://github.com/chrislusf/seaweedfs/issues/2386	2021-10-24 02:52:56 -07:00
Chris Lu	3be3c17f59	volume vacuum: avoid timeout with streaming progress report fix https://github.com/chrislusf/seaweedfs/issues/2396	2021-10-24 01:55:34 -07:00
Konstantin Lebedev	235329a92a	fix Tagging add xmlns	2021-10-20 19:40:22 +05:00
Konstantin Lebedev	dc7e525cb9	fix Tagging test	2021-10-20 19:12:00 +05:00
Konstantin Lebedev	0b834600d5	fix PutObjectTaggingHandler Unmarshal	2021-10-20 17:58:06 +05:00
Konstantin Lebedev	c78220a7f2	fix object tagging https://github.com/chrislusf/seaweedfs/issues/2389	2021-10-20 16:01:06 +05:00
Chris Lu	182f43ae5f	2.74	2021-10-18 14:23:54 -07:00
Chris Lu	309f46e3af	S3: fix upload limit if the size is more than 8GB	2021-10-18 14:14:18 -07:00
Chris Lu	cd4fa7561b	2.73	2021-10-18 10:47:48 -07:00
Chris Lu	a6bb509cb7	adjust mime detection	2021-10-18 10:47:39 -07:00
Konstantin Lebedev	dd2cc1acaf	fix https://github.com/chrislusf/seaweedfs/issues/2387	2021-10-18 16:27:57 +05:00
Chris Lu	97c963bac9	2.72	2021-10-17 17:40:27 -07:00
Chris Lu	3833dac3f7	continue to read from memory if there is no flush	2021-10-17 13:53:04 -07:00
Chris Lu	29fa1b9fdc	add logs	2021-10-17 13:50:34 -07:00
Chris Lu	cca62fdb30	mount: streaming renaming folders	2021-10-17 04:22:42 -07:00
Chris Lu	004e56c1a6	save updated lastTsNs	2021-10-17 02:01:20 -07:00
Chris Lu	71a8c80f6f	s3: continuously listen to file updates fix https://github.com/chrislusf/seaweedfs/issues/2342	2021-10-17 01:04:19 -07:00
Chris Lu	93bb7869b8	Revert "mount: fix renaming a deep directory with unvisited directories" This reverts commit `0ccdb937bb`.	2021-10-16 23:52:30 -07:00
Chris Lu	8e2c9713a3	turn on new faster algorithm to translate into visible chunks	2021-10-16 23:35:20 -07:00
Chris Lu	e7d1f183cc	ensure file id is populated	2021-10-16 23:34:27 -07:00
Chris Lu	fc1694b321	add tests	2021-10-16 23:34:12 -07:00
Chris Lu	0ccdb937bb	mount: fix renaming a deep directory with unvisited directories	2021-10-16 23:33:45 -07:00
Chris Lu	b9a2efd69b	temporarily reverting	2021-10-16 16:29:10 -07:00
Chris Lu	8a124ef9fc	Revert "remove deprecated code" This reverts commit `de7688c539`.	2021-10-16 16:09:33 -07:00
Chris Lu	93624635b4	Revert "go fmt" This reverts commit `6c704eb2ba`.	2021-10-16 16:09:30 -07:00
Chris Lu	6c704eb2ba	go fmt	2021-10-16 16:06:18 -07:00
Chris Lu	de7688c539	remove deprecated code	2021-10-16 16:06:12 -07:00
Chris Lu	7336990639	faster file read for large files	2021-10-16 16:03:16 -07:00
Chris Lu	8965a53c4d	add warning error	2021-10-16 15:57:30 -07:00
Chris Lu	8e3075d051	ensure file entry chunks have populated the file id	2021-10-16 01:05:48 -07:00
Chris Lu	822ee2475b	remove unnecessary file	2021-10-16 01:05:06 -07:00
Konstantin Lebedev	8de7915950	s3 mime detect	2021-10-14 15:03:11 +05:00
Chris Lu	5fd4b05c5e	Merge pull request #2381 from Juneezee/deprecate-ioutil refactor: move from io/ioutil to io and os package	2021-10-13 22:38:58 -07:00
Chris Lu	46a09c6074	adjust test	2021-10-13 22:38:47 -07:00
Eng Zer Jun	a23bcbb7ec	refactor: move from io/ioutil to io and os package The io/ioutil package has been deprecated as of Go 1.16, see https://golang.org/doc/go1.16#ioutil. This commit replaces the existing io/ioutil functions with their new definitions in io and os packages. Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>	2021-10-14 12:27:58 +08:00
Chris Lu	4cbd390fbe	test: add fail message	2021-10-13 20:42:20 -07:00
Chris Lu	b693a8d6b9	Merge pull request #2380 from kmlebedev/bucket_policy Bucket policy	2021-10-13 15:29:53 -07:00
Konstantin Lebedev	9d6ffa0ea1	GetBucketLifecycleConfigurationHandler	2021-10-14 01:35:33 +05:00
Konstantin Lebedev	458145425e	S3 MultipartUpload pass contentType to meta	2021-10-12 17:14:54 +05:00
Chris Lu	de1ec050e8	fier/s3: save "Content-Disposition" to extended properties related to https://github.com/chrislusf/seaweedfs/issues/2371	2021-10-12 00:04:28 -07:00
Chris Lu	34f764007f	escape file name only when necessary	2021-10-11 23:33:45 -07:00
Chris Lu	8e9273db99	s3: use "response-content-disposition" to overwrite default content-disposition fix one part of https://github.com/chrislusf/seaweedfs/issues/2371 see https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html	2021-10-11 23:29:52 -07:00
Chris Lu	f0b928ff5e	go fmt	2021-10-11 23:23:46 -07:00
Chris Lu	3afa451cdc	volume: find a non-empty offset when binary searching by timestamp	2021-10-11 22:00:41 -07:00
Chris Lu	b530f12327	volume: find a non-empty offset when binary searching by timestamp fix https://github.com/chrislusf/seaweedfs/issues/2364	2021-10-11 16:11:50 -07:00
Konstantin Lebedev	be4b3ed509	AclHandlers	2021-10-11 15:03:56 +05:00
Chris Lu	e6ef7b2387	return error early fix https://github.com/chrislusf/seaweedfs/issues/2370	2021-10-11 01:24:30 -07:00
Chris Lu	3d586be552	2.71	2021-10-10 22:40:44 -07:00
Chris Lu	f4676824a7	fix test code compilation	2021-10-10 22:14:13 -07:00
Chris Lu	8205166009	disable testing with redis server --- FAIL: TestNameList (0.00s) panic: exec: "redis-server": executable file not found in $PATH [recovered] panic: exec: "redis-server": executable file not found in $PATH goroutine 37 [running]: testing.tRunner.func1.2({0xde2f80, 0xc0003da160}) /opt/hostedtoolcache/go/1.17.1/x64/src/testing/testing.go:1209 +0x24e testing.tRunner.func1() /opt/hostedtoolcache/go/1.17.1/x64/src/testing/testing.go:1212 +0x218 panic({0xde2f80, 0xc0003da160}) /opt/hostedtoolcache/go/1.17.1/x64/src/runtime/panic.go:1038 +0x215 github.com/chrislusf/seaweedfs/weed/filer/redis3.TestNameList(0x407c59) /home/runner/work/seaweedfs/seaweedfs/weed/filer/redis3/kv_directory_children_test.go:45 +0x376 testing.tRunner(0xc0003e2680, 0xf57f48) /opt/hostedtoolcache/go/1.17.1/x64/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /opt/hostedtoolcache/go/1.17.1/x64/src/testing/testing.go:1306 +0x35a	2021-10-10 21:57:55 -07:00
Chris Lu	3e2acf677c	removing tikv to resolve "go mod tidy" problem tikv is causing "go mod tidy" problem. Need to resolve this before adding tikv back. go mod tidy go: finding module for package github.com/coreos/etcd/clientv3/balancer/picker go: finding module for package cloud.google.com/go/kms/apiv1 go: finding module for package github.com/coreos/etcd/clientv3/balancer/resolver/endpoint go: finding module for package google.golang.org/grpc/naming go: finding module for package github.com/coreos/etcd/clientv3/credentials go: finding module for package github.com/coreos/etcd/clientv3/balancer go: finding module for package github.com/d4l3k/messagediff go: found github.com/coreos/etcd/clientv3/balancer in github.com/coreos/etcd v3.3.26+incompatible go: found github.com/coreos/etcd/clientv3/balancer/picker in github.com/coreos/etcd v3.3.26+incompatible go: found github.com/coreos/etcd/clientv3/balancer/resolver/endpoint in github.com/coreos/etcd v3.3.26+incompatible go: found github.com/coreos/etcd/clientv3/credentials in github.com/coreos/etcd v3.3.26+incompatible go: found cloud.google.com/go/kms/apiv1 in cloud.google.com/go/kms v1.0.0 go: found github.com/d4l3k/messagediff in github.com/d4l3k/messagediff v1.2.1 go: finding module for package google.golang.org/grpc/naming github.com/chrislusf/seaweedfs/weed/filer/tikv imports github.com/tikv/client-go/v2/tikv imports go.etcd.io/etcd/clientv3 tested by go.etcd.io/etcd/clientv3.test imports github.com/coreos/etcd/integration imports github.com/coreos/etcd/proxy/grpcproxy imports google.golang.org/grpc/naming: module google.golang.org/grpc@latest found (v1.41.0), but does not contain package google.golang.org/grpc/naming	2021-10-10 19:27:02 -07:00
Chris Lu	bf218cd59d	removing etcd sequencer causing go mod tidy problem. If anyone wants this, please help to resolve this first. github.com/chrislusf/seaweedfs/weed/sequence imports go.etcd.io/etcd/client tested by go.etcd.io/etcd/client.test imports github.com/coreos/etcd/integration imports github.com/coreos/etcd/proxy/grpcproxy imports google.golang.org/grpc/naming: module google.golang.org/grpc@latest found (v1.41.0), but does not contain package google.golang.org/grpc/naming	2021-10-10 19:10:46 -07:00
Chris Lu	17e08a16f4	fix redis3 deletion	2021-10-09 04:54:48 -07:00
Chris Lu	ce1efeb0eb	adds more error message	2021-10-09 04:54:14 -07:00
Chris Lu	e4830bd93d	go fmt	2021-10-07 21:13:31 -07:00
Chris Lu	0a856241fe	avoid int bigger than math.MaxInt32 fix https://github.com/chrislusf/seaweedfs/issues/2363	2021-10-07 21:12:57 -07:00
Chris Lu	d688e10ed1	do not try to compress if not sure about the file content	2021-10-07 13:29:00 -07:00
Chris Lu	332f5ad3a8	revert temporary changes that disabled compression fix https://github.com/chrislusf/seaweedfs/issues/2362	2021-10-07 13:24:16 -07:00
Chris Lu	f3d8232e14	reduce one redis lookup on hot path	2021-10-06 22:01:19 -07:00
Chris Lu	2336a397dc	use pipeline to save some time	2021-10-06 20:40:18 -07:00
Chris Lu	371fead8a5	redis3 using redis native sorted set	2021-10-06 18:18:24 -07:00
Chris Lu	8668d49c9d	test with real redis cpu: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz BenchmarkNameList-12 1789 760599 ns/op BenchmarkRedis-12 17539 64122 ns/op PASS	2021-10-06 01:25:37 -07:00
Chris Lu	6b31f3c97a	add benchmark test	2021-10-06 00:37:57 -07:00
Chris Lu	893f0587b1	redis3 adds distributed locking	2021-10-06 00:03:54 -07:00
Chris Lu	f0d1e7bd05	skip ec volumes when loading normal volumes	2021-10-05 02:31:44 -07:00
Chris Lu	332d49432d	reduce concurrent volume grow requests	2021-10-05 01:58:30 -07:00
Chris Lu	96119eab00	refactor	2021-10-05 00:40:04 -07:00
Chris Lu	8a66306064	calculate disk usage in case of race condition related to https://github.com/chrislusf/seaweedfs/issues/2357	2021-10-04 23:32:07 -07:00
Chris Lu	4ed2994555	use tsMemory to determine whether read from disk or memory remove lastFlushTime	2021-10-04 16:02:56 -07:00
Chris Lu	6a030547a2	server: remove peer check if not starting master more fix https://github.com/chrislusf/seaweedfs/issues/2352	2021-10-04 03:27:10 -07:00
Chris Lu	62c2732fd1	add back logic to check master peers fix https://github.com/chrislusf/seaweedfs/issues/2352	2021-10-04 02:51:26 -07:00
Chris Lu	d4bb16e20e	Merge pull request #2354 from chrislusf/bptree Add Redis3	2021-10-04 02:43:15 -07:00
Chris Lu	947add39e6	clean up *SkipListElementReference loaded from Redis	2021-10-04 02:31:38 -07:00
Chris Lu	513fed323a	SkipListElementReference can be an empty object	2021-10-04 02:30:44 -07:00
Chris Lu	280ab7f95c	add test	2021-10-04 02:30:24 -07:00
Chris Lu	04662126bb	add redis3	2021-10-04 01:04:27 -07:00
Chris Lu	2b9aab3442	use 1000 per batch	2021-10-04 01:03:40 -07:00
Chris Lu	366f522a2d	add redis3	2021-10-04 01:01:31 -07:00
Chris Lu	ba7fbac07f	rename	2021-10-03 19:23:34 -07:00
Chris Lu	e6196cdc50	add name list	2021-10-03 17:54:25 -07:00
Chris Lu	a481c4a45e	return previous element if visited	2021-10-03 13:50:52 -07:00
Chris Lu	22d8684e88	refactor out listStore	2021-10-03 02:19:21 -07:00
Chris Lu	d343b0db57	update value	2021-10-03 01:15:14 -07:00
Chris Lu	4f50f8c2ca	insert key and value	2021-10-03 01:07:35 -07:00
Chris Lu	69b84bb771	TestFindGreaterOrEqual	2021-10-02 14:15:49 -07:00
Chris Lu	57e2fd3f9b	remove bptree	2021-10-02 14:03:54 -07:00
Chris Lu	4c1741fdbb	working skiplist	2021-10-02 14:02:56 -07:00
Chris Lu	b297849147	typo	2021-10-01 23:24:54 -07:00
Chris Lu	af207bbaf0	retry both assign volume and uploading data fix https://github.com/chrislusf/seaweedfs/issues/2351	2021-10-01 23:23:39 -07:00
Chris Lu	b6694279d7	Merge branch 'master' into bptree	2021-10-01 16:55:44 -07:00
Chris Lu	1e3fdf366f	go fmt	2021-10-01 12:10:24 -07:00
Chris Lu	e862b2529a	refactor	2021-10-01 12:10:11 -07:00
Chris Lu	88e006bc6d	Merge pull request #2349 from kmlebedev/fix_repl_volumes_per_step Topology update for every Nth id volume	2021-10-01 12:05:51 -07:00
Konstantin Lebedev	5e64b22b45	check that the topology has been updated	2021-10-01 18:51:22 +05:00
Chris Lu	f58ea6a2ee	add source name to error message	2021-10-01 02:19:30 -07:00
chrislusf	3ffbaaa071	use github.com/linxGnu/grocksdb	2021-10-01 03:16:03 +00:00
Konstantin Lebedev	2cecde89c3	rename opt volumesPerStep	2021-10-01 00:17:54 +05:00
Konstantin Lebedev	fc51ffce2b	https://github.com/chrislusf/seaweedfs/issues/1846	2021-09-30 20:24:24 +05:00
Chris Lu	a067deaabc	avoid possible modified location list fix issue 1 of https://github.com/chrislusf/seaweedfs/issues/2345	2021-09-28 16:54:18 -07:00
Chris Lu	040443e2d1	fix possible error case	2021-09-27 23:59:45 -07:00
Chris Lu	2e9372dcf7	volume stream read skips deleted content	2021-09-27 03:07:44 -07:00
Chris Lu	225b019fe0	stream read multiple volumes in a volume server	2021-09-27 02:51:31 -07:00
Chris Lu	1904448d4e	adjust starting offset	2021-09-27 02:01:29 -07:00
Chris Lu	5956a8b05a	adjust comment	2021-09-27 01:58:03 -07:00
Chris Lu	aa64f2ac4c	send needle data instead of raw needle body	2021-09-27 01:53:41 -07:00
Chris Lu	c4d7ee6c5c	volume server: read all files in a volume	2021-09-27 01:45:32 -07:00
Chris Lu	4a1d4d7462	s3: default to allow empty folder for better performance	2021-09-26 22:34:14 -07:00
Chris Lu	cee4d20bc1	2.70	2021-09-26 17:37:46 -07:00
Chris Lu	603ea2db73	avoid looping forever if there are no more metadata updates	2021-09-26 11:55:27 -07:00
Chris Lu	9887610b54	log tsNs should be processing time	2021-09-26 11:54:13 -07:00
Chris Lu	2baed2e1e9	avoid possible metadata subscription data loss Previous implementation append filer logs into one file. So one file is not always sorted, which can lead to miss reading some entries, especially when different filers have different write throughput.	2021-09-25 01:18:44 -07:00
Chris Lu	a814f3f0a8	adjust metadata tail output	2021-09-25 01:04:51 -07:00
Chris Lu	52fe86df45	use default 10000 for grpc port	2021-09-20 14:05:59 -07:00
Chris Lu	b3d88180ca	Merge branch 'master' into bptree	2021-09-19 23:56:59 -07:00
Chris Lu	fa7c65bd4b	2.69	2021-09-19 21:44:06 -07:00
Chris Lu	84fdda85e7	go fmt	2021-09-19 12:06:15 -07:00
Chris Lu	ad5099e570	refactor	2021-09-19 12:02:23 -07:00
Chris Lu	5abdc0be77	s3: avoid overwriting object with ACL/LegalHold/Retension/LockConfiguration requests	2021-09-19 03:24:47 -07:00
Chris Lu	ede7a65a50	update log	2021-09-19 01:33:50 -07:00
Chris Lu	c1255d30ad	add cache error log	2021-09-19 01:33:26 -07:00
Chris Lu	59dd271734	more glog	2021-09-19 00:29:51 -07:00
Chris Lu	4cbba2b1c6	add more glog to s3	2021-09-19 00:28:22 -07:00
Chris Lu	71175461ef	add glog for s3 handlers	2021-09-19 00:18:59 -07:00
Chris Lu	e066e2642c	add NodeStore	2021-09-18 15:32:17 -07:00
Chris Lu	198fa58e3c	Merge branch 'master' into bptree	2021-09-18 14:06:16 -07:00
Chris Lu	49d971e602	filer: redis store needs to clean up batch deleted sub folders	2021-09-18 14:05:16 -07:00
Chris Lu	8f2e4be074	wip	2021-09-18 14:04:30 -07:00
Chris Lu	b751debd31	split node based on the last inserted key	2021-09-18 01:29:47 -07:00
Chris Lu	2226c3c8b6	Merge branch 'master' into bptree	2021-09-17 10:35:21 -07:00
Chris Lu	94a01fcfcb	filer.remote.gateway: add options to include or exclude new bucket names to mirror	2021-09-15 23:04:16 -07:00
Chris Lu	b5f4910412	adjust help messages	2021-09-15 22:53:10 -07:00
Chris Lu	f0907eb83c	add command filre.remote.gateway	2021-09-15 22:48:04 -07:00
Chris Lu	63da4bbb54	separate filer.remote.gateway command to avoid confusion	2021-09-15 22:47:17 -07:00
Chris Lu	2789d10342	go fmt	2021-09-14 10:37:06 -07:00
Chris Lu	119d5908dd	shell: do not need to lock to see volume -h	2021-09-13 22:13:34 -07:00
Chris Lu	7504be58f9	Avoid xattr printed out as HTTP headers fix https://github.com/chrislusf/seaweedfs/issues/2336	2021-09-13 04:00:57 -07:00
Chris Lu	96514f0f00	fix tests	2021-09-13 02:19:48 -07:00
Chris Lu	20ac710ceb	2.68	2021-09-13 02:16:09 -07:00
Chris Lu	6cd1ce8b74	erasure coding: add cleanup step if anything goes wrong	2021-09-13 01:55:49 -07:00
Chris Lu	f74b29416a	better etag matching	2021-09-13 00:31:46 -07:00
Chris Lu	e9760f261e	minor	2021-09-13 00:31:06 -07:00
Chris Lu	e5fc35ed0c	change server address from string to a type	2021-09-12 22:47:52 -07:00
Chris Lu	2c9d4c8f43	custom grpc port: filer	2021-09-12 02:28:37 -07:00
Chris Lu	e690a2be16	custom grpc port: volume server	2021-09-12 02:25:15 -07:00
Chris Lu	232ad2fe65	custom grpc port: master follower	2021-09-12 02:25:00 -07:00
Chris Lu	bd1f800efe	custom grpc port: master	2021-09-12 02:19:10 -07:00
Chris Lu	7591336a22	log format	2021-09-11 14:27:57 -07:00
Chris Lu	8c6ff55226	add volume not found error type, to reduce error log	2021-09-11 14:26:41 -07:00
Chris Lu	7e600bff5f	adjust log level	2021-09-11 12:59:15 -07:00
Chris Lu	a243d7e047	EC: clean up if failed in the middle	2021-09-11 02:07:14 -07:00
Chris Lu	f7f01ac209	return if error	2021-09-11 02:06:43 -07:00
Chris Lu	59cb6146a3	refactor	2021-09-11 02:06:30 -07:00
Chris Lu	5496d68f6a	increase counter only if not early terminated	2021-09-11 02:05:55 -07:00
Chris Lu	2b8ab1534a	mysql table DDL: minor, not necessary, but a little more strict the directory is already hashed into dirhash. So this change is not strictly required.	2021-09-08 23:55:18 -07:00
joshuafc	5654d0d60d	CompactionTableSizeMultiplier of leveldb use default value. #2325 To improve performance of leveldb find key in condition of large directory(millions of files) which use uuid as filename.	2021-09-09 10:42:34 +08:00
Chris Lu	0b4269b6a8	remember commands even if failed	2021-09-08 15:55:19 -07:00
Chris Lu	0207f5fe9b	replicated remote.cache	2021-09-08 15:54:55 -07:00
Chris Lu	00edcf71fc	fix compilation	2021-09-08 07:27:03 -07:00
Chris Lu	574485ec69	better IP v6 support	2021-09-07 19:29:42 -07:00
Chris Lu	0128239c0f	handle ipv6 addresses	2021-09-07 16:43:54 -07:00
zhoub	6a7ed1bd0e	add bloom filter to leveldb_store to improve fuse performance.	2021-09-07 21:09:10 +08:00
Chris Lu	9fdf02bcda	remove detecting ipv6 Got this error on my local: transport: Error while dialing dial tcp [fe80::1]:19333: connect: no route to host related to https://github.com/chrislusf/seaweedfs/pull/2310	2021-09-07 02:31:34 -07:00
Chris Lu	889a39a24f	ipv6	2021-09-07 02:30:25 -07:00
Chris Lu	8c6d706328	2.67	2021-09-07 00:08:03 -07:00
Chris Lu	2b1feb732c	remote.cache supports replication	2021-09-06 18:30:44 -07:00
Chris Lu	3adc3da291	refactor	2021-09-06 16:35:55 -07:00
Chris Lu	6923af7280	refactoring	2021-09-06 16:20:49 -07:00
Chris Lu	64f6532fbe	update help message	2021-09-06 16:16:22 -07:00
Chris Lu	308d4b2dae	filer.remote.sync: createBucketWithRandomSuffix defaults to true	2021-09-06 15:14:26 -07:00
Chris Lu	8e4b43a017	minor	2021-09-06 15:13:38 -07:00
Chris Lu	c218ef20c7	filer.remote.sync: automatically detect the primary remote storage	2021-09-06 15:10:55 -07:00
Chris Lu	1702ce5395	remove unused code	2021-09-06 12:01:44 -07:00
Chris Lu	0cda61d539	return error if not found	2021-09-05 23:43:44 -07:00
Chris Lu	f62df7f695	fix wrong srcPath checking	2021-09-05 23:37:40 -07:00
Chris Lu	e93d4935e3	add other replica locations when assigning volumes	2021-09-05 23:32:25 -07:00
Chris Lu	7a13816e94	refactor	2021-09-05 23:17:15 -07:00
Chris Lu	6022db6d6a	2.66	2021-09-05 16:21:14 -07:00
Chris Lu	006c01a519	fix format	2021-09-05 16:18:50 -07:00
Chris Lu	65af3cf4df	master: disconnect only the phantom volume server fix https://github.com/chrislusf/seaweedfs/issues/2311	2021-09-05 15:20:03 -07:00
Chris Lu	57a95887d2	remote.cache remote.uncache supports all mounted directories	2021-09-05 14:47:06 -07:00
Chris Lu	28f45f8fa6	re-org	2021-09-05 14:38:31 -07:00
Chris Lu	c735608685	obfuscate secret keys on display	2021-09-05 14:23:49 -07:00
Chris Lu	32e94de86a	default auto trimming suffix to true	2021-09-05 13:27:35 -07:00
Chris Lu	60573fd3e2	option to map remote bucket to trimmed bucket name	2021-09-05 11:55:52 -07:00
Chris Lu	2348e8d8da	Merge pull request #2310 from nivekuil/ipv6 Detect ipv6 addresses	2021-09-05 10:56:44 -07:00
nivekuil	0fe9d2997b	Detect ipv6 addresses	2021-09-05 06:21:40 -07:00
Chris Lu	bdefdee4e6	filer.remote.sync: add option to add randomized suffix to buckets to avoid conflicts	2021-09-04 22:46:28 -07:00
Chris Lu	d57d4c5f8f	shell: add remote.mount.buckets	2021-09-04 21:37:25 -07:00
Chris Lu	98c68d7a79	filer.remote.sync fix upload logic	2021-09-04 18:46:28 -07:00
Chris Lu	d983aa4c7d	correct filtering	2021-09-04 13:58:14 -07:00
Chris Lu	c5ee03d6af	format	2021-09-04 13:57:55 -07:00
Chris Lu	cc278a5928	Update Makefile	2021-09-04 13:46:49 -07:00
Chris Lu	16a342eb8a	re-ordering	2021-09-04 13:46:44 -07:00
Chris Lu	796b2eb929	remember sync time	2021-09-04 13:46:22 -07:00
Chris Lu	53b9b521c9	adjust error message	2021-09-04 13:46:06 -07:00
Chris Lu	49952ba905	fix cache/uncache filters	2021-09-04 13:45:39 -07:00
Chris Lu	99d4594ee8	update remote storage mount mapping	2021-09-04 05:22:34 -07:00
Chris Lu	df29281536	rename	2021-09-04 05:11:06 -07:00
Chris Lu	c3db389e42	refactor	2021-09-04 05:09:38 -07:00
Chris Lu	e9ce099453	rename	2021-09-04 05:03:51 -07:00
Chris Lu	8707ef00a6	refactor	2021-09-04 05:02:20 -07:00
Chris Lu	13cc2ef723	when creating bucket, use remote storage client for default storage	2021-09-04 04:54:55 -07:00
Chris Lu	63c84584bb	refactor	2021-09-04 04:53:36 -07:00
Chris Lu	4b28c5f6c3	filer.remote.sync: split into buckets mode and single directory mode	2021-09-04 04:35:46 -07:00
Chris Lu	da49d25950	auto list of storage types	2021-09-04 00:18:21 -07:00
Chris Lu	38f73c8324	add gcs project id	2021-09-03 23:38:33 -07:00
Chris Lu	8ec357b3d3	go mod	2021-09-03 23:25:33 -07:00
Chris Lu	bbc77f7af4	fix compilation	2021-09-03 22:56:59 -07:00
Chris Lu	0652805236	cloud drive: add createBucket() deleteBucket()	2021-09-03 22:30:55 -07:00
Chris Lu	03a31587ce	go fmt	2021-09-03 20:42:28 -07:00
Chris Lu	83cd0fc739	cloud drive: add list buckets	2021-09-03 20:42:02 -07:00
Chris Lu	fbfc90fd1e	adjust formatting remote location	2021-09-03 18:52:37 -07:00
Chris Lu	491f7636f8	s3: CopyObject return http Status 400 Bad Request for non-existing source fix https://github.com/chrislusf/seaweedfs/issues/2306	2021-09-03 17:38:39 -07:00
Chris Lu	bca4a9de78	simplify	2021-09-02 23:09:24 -07:00
Chris Lu	958125bd02	conforming to http user agent common practice	2021-09-02 22:55:35 -07:00
Chris Lu	11a496404b	reset wait time	2021-09-02 19:55:01 -07:00
Chris Lu	7f2a8246a9	minor	2021-09-02 12:32:59 -07:00
Chris Lu	1b942dfa3c	add error message	2021-09-02 12:32:46 -07:00
Chris Lu	9a73b0e3c9	refactor	2021-09-02 07:07:16 -07:00
Chris Lu	e281f0fa82	refactor	2021-09-02 06:53:21 -07:00
Chris Lu	24fda6f47f	remove QR code for mobile upload	2021-09-01 16:59:37 -07:00
Chris Lu	7ce97b59d8	go fmt	2021-09-01 02:45:42 -07:00
Chris Lu	d1a4e19a3f	volume: copy file also copies modification time to ensure ttl can work well	2021-09-01 02:42:57 -07:00
Chris Lu	3bd48c4f29	filer.remote.sync: exit when directory is unmounted this will not propagate the deletions back to the cloud	2021-09-01 01:29:22 -07:00
Chris Lu	3faaa6e360	ensure cached client with updated storage conf	2021-09-01 01:27:45 -07:00
Chris Lu	8e125339d5	line wrap	2021-08-31 23:30:28 -07:00
Chris Lu	68ea99190f	add additionalPathPrefixes	2021-08-31 23:26:26 -07:00
Chris Lu	43fd11278e	support follow additional path prefixes	2021-08-31 23:23:08 -07:00
Chris Lu	cca588429b	shell: fs.meta.save skip saving system logs	2021-08-31 23:05:56 -07:00
Chris Lu	0ee208c1a4	adjust example	2021-08-31 02:04:32 -07:00
Chris Lu	97b7948cd9	adjust error message	2021-08-31 01:07:34 -07:00
Chris Lu	c9205458a1	cloud drive: create mount directory if not exists	2021-08-31 01:07:19 -07:00
Chris Lu	1dfcdc53b7	cloud drive: add storj, filebase	2021-08-30 17:43:14 -07:00
Chris Lu	701cc133f3	cloud drive: s3 add option for v4 signature	2021-08-30 17:28:33 -07:00
Chris Lu	f117f7bf54	adjust comment	2021-08-30 15:56:14 -07:00
Chris Lu	72d4ff6b28	Merge pull request #2280 from blacktear23/filerstore-tikv Use TiKV as Filerstore	2021-08-30 02:21:14 -07:00
Chris Lu	f811fd0903	fix build	2021-08-30 02:03:08 -07:00
yulai.li	b17b81529e	Add build tags for TiKV filerstore	2021-08-30 15:59:25 +08:00
Rain Li	e5f9ff983d	Merge branch 'master' into filerstore-tikv	2021-08-30 15:52:54 +08:00
Chris Lu	44d0ac95f6	add cluster id if already set	2021-08-29 22:19:46 -07:00
Chris Lu	18bf556d93	fix comments	2021-08-29 22:19:25 -07:00
Chris Lu	21f37a0a75	remove unused code	2021-08-29 22:19:11 -07:00
Chris Lu	5adfdd7982	change to a new bloom filter library	2021-08-29 21:26:38 -07:00
Chris Lu	05f32376eb	add cluster id in filer configuration response	2021-08-29 21:02:10 -07:00
Chris Lu	eacaa44dc2	refactor	2021-08-29 19:13:48 -07:00
Chris Lu	6deee4c0b9	import hdfs	2021-08-29 18:49:43 -07:00
Chris Lu	a31f2907f0	cloud drive: filer.remote.sync supports remove folder	2021-08-29 18:46:28 -07:00
Chris Lu	001a472057	cloud mount: remote storage support hdfs	2021-08-29 18:41:29 -07:00
Chris Lu	05d2774117	refactoring	2021-08-28 22:49:38 -07:00
Chris Lu	975fbc9ee3	add wasabi example	2021-08-28 22:49:38 -07:00
Chris Lu	bec3f63298	2.65	2021-08-28 05:27:33 -07:00
Chris Lu	223c1b3571	adjust invalidation logic	2021-08-28 05:21:01 -07:00
nivekuil	8c523d50ad	Invalidate mount meta cache more aggressively	2021-08-28 04:26:37 -07:00
Chris Lu	65a29ad42a	add warning on unmount a folder	2021-08-28 02:25:48 -07:00
Chris Lu	a22f37b01c	remove tencent region	2021-08-28 02:23:03 -07:00
Chris Lu	a9b41900d8	handle cases when mounted directory does not exist	2021-08-28 01:56:56 -07:00
Chris Lu	0d8936f6cd	add aliyun example	2021-08-28 01:09:48 -07:00
Chris Lu	dc481c081c	remove unused function	2021-08-27 22:18:54 -07:00
Chris Lu	9242f3aaeb	fix build	2021-08-26 17:33:57 -07:00
Chris Lu	49a8dfb976	adjust default concurrent level	2021-08-26 17:05:56 -07:00
Chris Lu	6a0bb7106b	cloud drive: parallelize remote storage downloading	2021-08-26 16:16:26 -07:00
Chris Lu	05a648bb96	refactor: separating out remote.proto	2021-08-26 15:18:34 -07:00
yulai.li	c1dc5ab4ac	Add deleterange_concurrency to filer configuration file	2021-08-26 18:25:08 +08:00
yulai.li	318757ef8c	Change DeleteFolderChildren to DeleteRange api	2021-08-26 17:49:56 +08:00
yulai.li	de8ef28460	Update filer.toml scaffold to support tikv store	2021-08-26 16:26:27 +08:00
yulai.li	546efeba8f	Fix build bug	2021-08-26 16:20:35 +08:00
yulai.li	2088f28424	init post	2021-08-26 15:20:18 +08:00
Chris Lu	c08ac536ed	cloud drive: add support for Wasabi * disable md5, sha256 checking to avoid reading one chunk twice * single threaded upload to avoid chunk swapping (to be enhanced later)	2021-08-25 17:34:29 -07:00
Chris Lu	9bcf94b2b1	ensure multi-threaded correctness	2021-08-25 17:28:50 -07:00
Chris Lu	a7a914f120	cloud drive: add support for Baidu BOS	2021-08-24 23:46:33 -07:00
Chris Lu	2d805beed4	add help mesages	2021-08-24 23:28:35 -07:00
Chris Lu	c6c97bd83d	add default env variables	2021-08-24 23:25:36 -07:00
Chris Lu	a19c728034	cloud drive: add support for Tencent COS	2021-08-24 23:19:45 -07:00
Chris Lu	19a81d25af	cloud drive: add support for Aliyun OSS	2021-08-24 23:14:24 -07:00
Chris Lu	47d775cf68	cloud drive: add support for BackBlaze	2021-08-24 22:30:06 -07:00
Chris Lu	6bab20d862	cat entry content	2021-08-24 02:38:32 -07:00
Chris Lu	e9ebe24f2e	cloud drive: add support for Azure	2021-08-24 01:18:30 -07:00
Chris Lu	7c39a18ba5	update azure library	2021-08-24 00:32:35 -07:00
Chris Lu	a78d0227cd	adjust package name	2021-08-23 23:19:31 -07:00
Chris Lu	12631a3f5b	cloud drive: gcs simplify a little bit	2021-08-23 14:43:01 -07:00
Chris Lu	00c4e06caa	cloud drive: s3 configurable force path style	2021-08-23 03:30:41 -07:00
Chris Lu	f0cc130849	do not force path style for better compatibility	2021-08-23 03:09:41 -07:00
Chris Lu	2836a58d87	cloud drive: S3 supports storage class	2021-08-23 02:18:59 -07:00
Chris Lu	95e2b83ca5	fix format	2021-08-23 00:49:59 -07:00
Chris Lu	ff7dc3b44c	2.64	2021-08-23 00:39:15 -07:00
Chris Lu	ca35a77e85	adjust help message	2021-08-23 00:37:55 -07:00
Chris Lu	258063de26	cloud drive: add google cloud storage	2021-08-23 00:29:27 -07:00
Chris Lu	df1d6133a8	bptree does not work well for auto-increasing keys	2021-08-22 18:19:26 -07:00
Chris Lu	3b2c39f1be	mount: set name when mount path equals mount folder fix https://github.com/chrislusf/seaweedfs/issues/2275#issuecomment-903255876	2021-08-22 04:59:09 -07:00
Chris Lu	51c8f2518f	change key type to ItemKey	2021-08-21 15:54:42 -07:00
Chris Lu	b3e49d2758	change value type to ItemValue	2021-08-21 15:52:17 -07:00
Chris Lu	38c8470d1d	add back non_dedup	2021-08-21 15:13:13 -07:00
Chris Lu	849f185a20	add memory kv store	2021-08-21 15:00:44 -07:00
Chris Lu	5f6cc9a814	make proto node	2021-08-21 13:36:52 -07:00
Chris Lu	2158d4fe4d	adjust help message	2021-08-21 02:17:10 -07:00
Chris Lu	172da83449	bpnode use get prev and next	2021-08-20 18:50:16 -07:00
Chris Lu	01661ec6a7	move to getter setter file	2021-08-20 18:37:34 -07:00
Chris Lu	0c360eb6b2	add getter and setter for root of tree and map	2021-08-20 18:34:50 -07:00
Chris Lu	1b1ab331f6	Merge pull request #2274 from qieqieplus/gzip-pool	2021-08-20 04:22:41 -07:00
Chris Lu	88d68cad87	remove dedup	2021-08-20 04:14:52 -07:00
qieqieplus	7720533f84	reduce gzip allocation	2021-08-20 18:38:18 +08:00
Chris Lu	2d237da74a	remove size since each put/get will have to update the root node	2021-08-20 01:19:11 -07:00
Chris Lu	ec72547c8d	started by copying from https://sourcegraph.com/github.com/timtadh/data-structures@master/-/tree/tree/bptree	2021-08-20 01:12:52 -07:00
Chris Lu	05fc7db755	filer.copy: large files pack file ids into chunks	2021-08-18 23:46:54 -07:00
mauroparente	c63713774b	Fix image resize https://github.com/chrislusf/seaweedfs/issues/2262	2021-08-18 23:15:35 +02:00
Chris Lu	570d2eccec	skip body if not allowed by http status	2021-08-18 00:56:35 -07:00
Chris Lu	ccb3df41f8	elide secret key	2021-08-17 11:27:08 -07:00
Konstantin Lebedev	865b06b7fe	The IEEE assembler optimizations has been submitted and will be part of the Go 1.6 standard library. https://github.com/klauspost/crc32	2021-08-17 13:06:48 +05:00
Chris Lu	e2aa3cf63b	fix go test	2021-08-15 23:20:46 -07:00
Chris Lu	1f35d32be0	2.63	2021-08-15 23:14:59 -07:00
Chris Lu	40dc283b2d	fix locating data chunks	2021-08-15 23:07:58 -07:00
Chris Lu	72eb6d5b9d	ensure no writes to remote storage if content is not changed	2021-08-15 20:23:41 -07:00
Chris Lu	5a7c40510f	format output	2021-08-15 20:07:13 -07:00
Chris Lu	bb94930196	add some delays if error	2021-08-15 20:06:47 -07:00
Chris Lu	5d5a21ba2d	adjust log format	2021-08-15 19:46:45 -07:00
Chris Lu	c45c12aedc	metadata updates	2021-08-15 19:27:30 -07:00
Chris Lu	507411bdf0	fix https://github.com/chrislusf/seaweedfs/issues/2260 fix https://github.com/chrislusf/seaweedfs/issues/2260	2021-08-15 19:16:00 -07:00
Chris Lu	c3ffd457ef	fix compilation error	2021-08-15 12:40:22 -07:00
Chris Lu	49b5e47bd1	retry forever with filer.remote.sync, and some refactoring	2021-08-15 12:38:26 -07:00
Chris Lu	fda2fc47b1	add RetryForever	2021-08-15 12:37:35 -07:00
Chris Lu	a539d64896	refactor	2021-08-15 12:09:54 -07:00
Chris Lu	8f7d2d317f	readerAt need to use the right offset fix https://github.com/chrislusf/seaweedfs/issues/2259	2021-08-15 11:55:58 -07:00
Chris Lu	0b2a92d371	skip if http.StatusNoContent	2021-08-15 11:27:49 -07:00
Chris Lu	ec989b0377	remove println	2021-08-15 02:10:27 -07:00
Chris Lu	9462f5129a	shell: add "remote.meta.sync"	2021-08-15 01:53:46 -07:00
Chris Lu	3ada61c875	rename	2021-08-14 21:50:35 -07:00
Chris Lu	c34747c79d	rename, fix wrong logic.	2021-08-14 21:46:34 -07:00
Chris Lu	cb53802752	adjust help message	2021-08-14 15:55:53 -07:00
Chris Lu	889b143fa7	adjust modification detection logic	2021-08-14 15:44:47 -07:00
Chris Lu	f365af81c2	parallelize remote content fetching	2021-08-14 15:41:37 -07:00
Chris Lu	53e66980b2	add comments	2021-08-14 15:16:10 -07:00
Chris Lu	9921801e0c	Revert "use default or path-specific setting for cache replication level" This reverts commit `ba6923b223`.	2021-08-14 15:14:26 -07:00
Chris Lu	ba6923b223	use default or path-specific setting for cache replication level	2021-08-14 15:14:01 -07:00
Chris Lu	708debca14	remote.cache and uncache: more flexible options to select files to cache or uncache	2021-08-14 15:11:55 -07:00
Chris Lu	08258a819d	fix mistake	2021-08-14 05:10:30 -07:00
Chris Lu	4909bd9684	gRpc connection error on filer when no volume left #2243 fix https://github.com/chrislusf/seaweedfs/issues/2243 grpc do not cache connections only when connection problem happens. Normal error results should not close the shared grpc connection.	2021-08-14 05:06:44 -07:00
Chris Lu	8126ab4b5d	rename	2021-08-14 05:03:45 -07:00
Chris Lu	0db2517994	go fmt	2021-08-14 02:55:44 -07:00
Chris Lu	7937db52e1	Filer locationPrefix configure does not exec replication #2257 fix https://github.com/chrislusf/seaweedfs/issues/2257	2021-08-14 02:54:13 -07:00
Chris Lu	0c66b173a4	fix	2021-08-13 11:31:43 -07:00
Chris Lu	e02a317d3d	adjust retry logic in case some data is partially written	2021-08-13 11:30:38 -07:00
Chris Lu	2d519c6cb6	adjust the retry logic	2021-08-13 11:13:30 -07:00
Chris Lu	b961fcd338	filer: stream read from volume server, reduce memory usage	2021-08-13 11:00:11 -07:00
Chris Lu	f4decf02df	volume copying: clean up stale volume data files fix https://github.com/chrislusf/seaweedfs/issues/2250	2021-08-13 03:24:21 -07:00
Chris Lu	0f7d4556d8	shell: volume.tier.move makes up changes if volume move failed	2021-08-13 03:09:28 -07:00
Chris Lu	333cdce485	add verbose message	2021-08-13 02:57:14 -07:00
Chris Lu	78e8ddf910	Only when tailing volume, the zero-ed cookie should skip checking. This only happens when checkCookie == false and fsync == false.	2021-08-13 02:09:35 -07:00
Chris Lu	a8617c1a39	tail volume: fix zero cookie problem from batch deletion	2021-08-13 01:54:35 -07:00
Chris Lu	5a0f92423e	use grpc and jwt	2021-08-12 21:40:33 -07:00
Chris Lu	6238644c35	remove gateway command	2021-08-12 20:52:04 -07:00
Chris Lu	d1d1fc772c	move some volume lookup operations to grpc jwt related lookup will come in next commit	2021-08-12 20:33:00 -07:00
Chris Lu	5571f4f70a	master: add master.follower to handle read file id lookup requests	2021-08-12 18:10:59 -07:00
Chris Lu	5469019852	adjust data type	2021-08-12 17:54:34 -07:00
Chris Lu	bfac55e6c0	avoid integer overflow fix https://github.com/chrislusf/seaweedfs/issues/2254	2021-08-11 22:22:49 -07:00
Chris Lu	5516fa8e80	Merge pull request #2251 from kmlebedev/rertyUpdateIamIdentity Retry save and update IAM identity	2021-08-11 09:21:03 -07:00
Konstantin Lebedev	f0afd35eec	Retry save and update IAM identity https://github.com/chrislusf/seaweedfs/issues/2242	2021-08-11 19:29:04 +05:00
byunghwa.yun	775dfbae85	Synchronize number of open files	2021-08-11 23:14:56 +09:00
Konstantin Lebedev	ec09966fd3	Retry save and update IAM identity https://github.com/chrislusf/seaweedfs/issues/2242	2021-08-11 17:53:48 +05:00
Chris Lu	f2cd753bf9	fix avoid lock error fix https://github.com/chrislusf/seaweedfs/issues/2247	2021-08-10 14:34:13 -07:00
Chris Lu	01336d71eb	minor	2021-08-10 13:04:33 -07:00
Chris Lu	1154e23e2d	add logs for volume creation	2021-08-10 13:04:25 -07:00
Chris Lu	057ef429ac	format	2021-08-10 12:33:29 -07:00
Chris Lu	b63b042afc	dedup keeps the largest replica	2021-08-10 12:30:41 -07:00
Chris Lu	0526db12e2	do not treat read only volumes differently	2021-08-10 11:37:12 -07:00
Chris Lu	e50a5b8e28	minor: print disk type	2021-08-10 11:10:09 -07:00
Chris Lu	db6275a0c8	print out balance ratio	2021-08-10 10:43:42 -07:00
Chris Lu	9d85569c55	ensure using local quorum consistency	2021-08-10 05:10:57 -07:00
Chris Lu	85832d02c0	wait for goroutines	2021-08-10 04:13:12 -07:00
Bl1tz23	1c94b3d013	merge master, resolve conflicts	2021-08-10 13:45:24 +03:00
Bl1tz23	e6e57db530	Add liveness\readiness probe for s3 api handler on /status path	2021-08-10 13:42:46 +03:00
Chris Lu	8ff6c9a0c6	output format	2021-08-10 03:25:18 -07:00
Chris Lu	48f448ee09	parallelize tier move	2021-08-10 03:08:29 -07:00
Chris Lu	69a6da7969	avoid fail on tail error	2021-08-10 02:50:28 -07:00
Chris Lu	18228f3044	fix help message	2021-08-10 02:48:41 -07:00
Chris Lu	69655ba8e5	mount: cache on reading remote storage	2021-08-09 22:11:57 -07:00
Chris Lu	a7012d9729	fix	2021-08-09 16:03:03 -07:00
Chris Lu	02f728cb54	fix bug with remote.uncache	2021-08-09 15:24:21 -07:00
Chris Lu	8d3e275735	remote: filer cache remote content on read	2021-08-09 15:16:45 -07:00
Chris Lu	9096f6f4f7	cache: set upper limit of chunk size	2021-08-09 15:08:53 -07:00
Chris Lu	402315f117	go fmt	2021-08-09 14:37:34 -07:00
Chris Lu	a6be2520c9	fix	2021-08-09 14:37:25 -07:00
Chris Lu	713c035a6e	shell: remote.cache remote.uncache	2021-08-09 14:35:18 -07:00
Chris Lu	8cfd487608	2.62	2021-08-08 23:33:12 -07:00
Chris Lu	734c980040	volume: support concurrent download data size limit	2021-08-08 23:25:16 -07:00
Chris Lu	96ce85f5ae	rename	2021-08-08 22:33:31 -07:00
Chris Lu	c5f38c365d	go fmt	2021-08-08 22:30:36 -07:00
Chris Lu	df85f7a1eb	adjust help message	2021-08-08 22:30:12 -07:00
Chris Lu	c0b12da4ef	shell: add filer.remote.unmount	2021-08-08 22:26:37 -07:00
Chris Lu	882a93dacd	fix tests	2021-08-08 17:56:26 -07:00
Chris Lu	7412ccdf88	write back remote entry to local entry after uploading to remote	2021-08-08 17:55:03 -07:00
Chris Lu	dcf614a8c3	skip if the remote entry update is because of internal managerial operations	2021-08-08 15:58:10 -07:00
Chris Lu	eed26af266	Merge branch 'master' into add_remote_storage	2021-08-08 15:48:04 -07:00
Chris Lu	4370a4db63	use int64 for volume count in case of negative overflow	2021-08-08 15:19:39 -07:00
Chris Lu	26c222f596	shell: volume.tier.move avoid moving all volumes to one destination	2021-08-08 15:12:39 -07:00
Chris Lu	13e45e1605	filer.remote.sync can work now	2021-08-08 01:21:42 -07:00
Chris Lu	8f5170c138	remove imports	2021-08-07 16:20:17 -07:00
Chris Lu	46b0cb8c86	Merge branch 'master' into add_remote_storage	2021-08-07 15:41:27 -07:00
Chris Lu	de730b079d	ChunkStreamReader implenents io.ReaderAt	2021-08-07 15:41:07 -07:00
Chris Lu	59732a0529	refactoring	2021-08-07 15:35:27 -07:00
Chris Lu	ecb234f75a	refactor	2021-08-07 14:46:23 -07:00
Chris Lu	270770d7d7	refactor	2021-08-07 14:18:53 -07:00
Chris Lu	679f800caa	Merge branch 'master' into add_remote_storage	2021-08-06 20:27:12 -07:00
Chris Lu	0c0f77e2ae	skip not found error on deletion	2021-08-06 19:35:47 -07:00
Chris Lu	8828f485c0	print volume deletion error	2021-08-06 19:30:22 -07:00
Chris Lu	d2b23f3d38	Merge pull request #2238 from combineads/change_default_mode Change default permissions	2021-08-05 21:28:06 -07:00
Chris Lu	e23ca3845b	Merge pull request #2237 from combineads/add_mtime Add crtime and mtime	2021-08-05 21:27:13 -07:00
Chris Lu	286e5dd375	Merge branch 'master' into add_remote_storage	2021-08-05 21:07:04 -07:00
Chris Lu	1e22166939	adjust error message	2021-08-05 21:06:55 -07:00
byunghwa.yun	f3dc909b21	Change default permissions	2021-08-06 12:24:35 +09:00
byunghwa.yun	e84fad9acf	Add crtime and mtime	2021-08-06 12:13:37 +09:00
Chris Lu	1a5d29520c	add tests	2021-08-05 14:49:24 -07:00
Chris Lu	6b743dbbf9	refactor client subscribe metadata	2021-08-04 16:25:46 -07:00
Chris Lu	b9ecf1e3a8	refacotring	2021-08-04 14:56:13 -07:00
Chris Lu	d84c311699	refactoring	2021-08-04 12:30:18 -07:00
Chris Lu	f6a9ad8001	fix tests	2021-08-04 00:31:06 -07:00
Chris Lu	42969c9c62	Merge branch 'master' into add_remote_storage	2021-08-03 21:20:05 -07:00
Chris Lu	8cf0c515bf	shell: volume.fix.replication retries even when there is no error #2235 fix https://github.com/chrislusf/seaweedfs/issues/2235	2021-08-03 09:32:55 -07:00
Chris Lu	d2ddf1dbdb	typo	2021-08-03 09:32:31 -07:00
Chris Lu	3afbf04007	add TLS grpc support for filer meta clients	2021-08-03 01:30:35 -07:00
Chris Lu	fe60c6ef9a	minor changes	2021-08-02 11:49:40 -07:00
byunghwa.yun	bdc7730fdb	Add autocomplete	2021-08-03 02:56:52 +09:00
Chris Lu	2ae9705442	adjust text	2021-08-01 22:55:19 -07:00
Chris Lu	3739717092	Revert "adds a test" This reverts commit `f690643b47`.	2021-08-01 22:54:45 -07:00
Chris Lu	f690643b47	adds a test	2021-08-01 22:53:50 -07:00
Chris Lu	6de786185d	volume.balance: balance read only volumes first	2021-08-01 22:32:50 -07:00
Chris Lu	3bb640b786	add -force option following #2228	2021-08-01 20:03:05 -07:00
byunghwa.yun	9e839cb5cc	Add force option in volume.deleteEmpty command	2021-08-02 11:37:20 +09:00
Chris Lu	56ee1d5ef1	2.61	2021-08-01 15:50:19 -07:00
Chris Lu	81b255df8b	Merge branch 'master' into add_remote_storage	2021-08-01 15:44:21 -07:00
Chris Lu	58bc3ecf47	add default quietFor value	2021-08-01 15:36:06 -07:00
Chris Lu	2ca1839d77	shell: add `volume.deleteEmpty` command	2021-08-01 15:33:45 -07:00
Chris Lu	89933c46d2	s3: skip hidden directories in /buckets folder	2021-08-01 12:28:08 -07:00
Chris Lu	1c7e404abe	remove buckets folder option the related code still works for old deployments	2021-08-01 12:23:16 -07:00
Chris Lu	cb1dbd3135	refactor	2021-08-01 11:53:46 -07:00
Chris Lu	1161c70dab	Merge branch 'master' into add_remote_storage	2021-08-01 01:10:13 -07:00
Chris Lu	9cc84a910f	volume: deletion can skip volume size checking fix https://github.com/chrislusf/seaweedfs/issues/2225	2021-08-01 00:32:51 -07:00
Chris Lu	767edd3c08	rename	2021-07-31 23:52:09 -07:00
Chris Lu	9df7d16791	read <- remote_storage	2021-07-31 22:39:38 -07:00
Chris Lu	f5a69a0e44	Merge branch 'master' into add_remote_storage	2021-07-31 09:39:48 -07:00
Chris Lu	6ba65c3382	customizable debug port	2021-07-31 09:18:41 -07:00
byunghwa.yun	5d4438a72f	Fix typo	2021-07-31 22:23:52 +09:00
Chris Lu	1ff8285d82	debug from any server	2021-07-31 02:13:21 -07:00
Chris Lu	0d6f45cb46	add debug option to generate full goroutine stack dump	2021-07-31 02:00:01 -07:00
Chris Lu	a3290faf17	shell command to calculate size / number of volumes in a collection fix https://github.com/chrislusf/seaweedfs/issues/2224	2021-07-30 15:49:46 -07:00
Chris Lu	c6f992b2a3	remove dead code	2021-07-30 15:18:01 -07:00
Chris Lu	899963ac20	remote storage location changed to struct	2021-07-29 02:08:55 -07:00
Chris Lu	c090d6bb25	add ReadRemote(), add read remote setup when filer starts	2021-07-28 22:43:12 -07:00
Chris Lu	4deac06da0	log fix	2021-07-28 22:25:04 -07:00
divanikus	5c6270a93a	shell: ability to use wildcards for collections, all collections if ommited	2021-07-27 23:53:01 +03:00
Chris Lu	035b0bae29	refactor	2021-07-27 03:32:24 -07:00
Chris Lu	1752eeb538	remote.mount saves the mapping	2021-07-27 03:26:35 -07:00
Chris Lu	4b94b03d90	directory to remote storage mapping	2021-07-27 01:16:28 -07:00
Chris Lu	99b599aa8a	remote.mount	2021-07-26 22:53:44 -07:00
Chris Lu	5dede5d38d	2.60	2021-07-25 22:09:09 -07:00
Chris Lu	35f70c51b0	refactor	2021-07-25 03:37:37 -07:00
Chris Lu	72eec84167	shell: add fs.mkdir	2021-07-25 02:37:39 -07:00
byunghwa.yun	2b28a818f1	Fix mysql sql for batch delete	2021-07-25 11:06:14 +09:00
Chris Lu	10464f47d0	Merge pull request #2217 from combineads/add_webp_format	2021-07-24 09:42:41 -07:00
byunghwa.yun	7374b5b473	Add webp extension for resizing	2021-07-24 14:32:31 +09:00
byunghwa.yun	2595f269d1	Add the webp image type	2021-07-24 14:26:40 +09:00
Chris Lu	ac28611817	snowflake sequencer need an unique id fix https://github.com/chrislusf/seaweedfs/issues/2213	2021-07-23 20:54:03 -07:00
Chris Lu	5c14da0f1e	filer.copy: fail early if assign request failed fix https://github.com/chrislusf/seaweedfs/issues/2216	2021-07-23 20:01:43 -07:00
Chris Lu	4cc2165061	weed -h subcommand list is not sorted, makes discovery difficult fix https://github.com/chrislusf/seaweedfs/issues/2214	2021-07-23 18:44:53 -07:00
Chris Lu	10fc478557	scaffold -config= should output to stdout fix https://github.com/chrislusf/seaweedfs/issues/2212	2021-07-23 18:41:25 -07:00
Chris Lu	092932af8f	shell: faster bucket delete avoid waiting for large object metadata	2021-07-23 11:25:46 -07:00
Chris Lu	bdb632fa62	Merge pull request #2211 from qieqieplus/filer-bloom-filter add bloom filter for filer leveldb/rocksdb	2021-07-22 23:35:54 -07:00
qieqieplus	2b46df13f8	add bloom filter for filer leveldb/rocksdb	2021-07-23 14:05:59 -06:00
Chris Lu	60f5c0a2f5	fix security alert on github.com/dgrijalva/jwt-go resolve https://github.com/chrislusf/seaweedfs/security/dependabot/go.mod/github.com%2Fdgrijalva%2Fjwt-go/open	2021-07-22 10:22:09 -07:00
Chris Lu	182288f860	filer: fix mysql, postgres batch delete error	2021-07-22 08:23:20 -07:00
Chris Lu	84d91f143f	mount: hide /etc folder also from mount	2021-07-22 00:40:16 -07:00
Chris Lu	7359193e97	go fmt	2021-07-21 14:38:12 -07:00
Chris Lu	cc32436d63	Merge pull request #2209 from combineads/fuse_option Fix error handling	2021-07-21 12:37:20 -07:00
Chris Lu	67537ff05b	Merge pull request #2208 from combineads/add_rm_options	2021-07-21 11:08:43 -07:00
byunghwa.yun	e0c7708b03	Fix error handling	2021-07-21 22:48:04 +09:00
byunghwa.yun	41d559f54e	Add fs rm options	2021-07-21 21:10:36 +09:00
Chris Lu	70effac0d3	configure and store remote configurations	2021-07-21 02:24:34 -07:00
Chris Lu	3138805b33	Merge pull request #2200 from combineads/add_fs_rm weed shell: fs.rm remove file or directory	2021-07-21 00:35:54 -07:00
Chris Lu	a125c8fbe2	Merge pull request #2204 from nivekuil/tokenpolicy cassandra: use LocalQuorum for all queries	2021-07-20 15:37:31 -07:00
nivekuil	b9a67d46c5	cassandra: use LocalQuorum for all queries This changes this filer store from eventual to strong consistency at the cost of read performance.	2021-07-20 14:47:39 -07:00
Chris Lu	54c8bc8673	Merge branch 'remote_overlay'	2021-07-19 23:18:35 -07:00
Chris Lu	7ab389e7ec	optimization: improve random range query for large files	2021-07-19 23:07:22 -07:00
Chris Lu	93c37cfded	Update filechunk_manifest.go	2021-07-19 18:41:41 -07:00
byunghwa.yun	b23b307e08	Add fs rm	2021-07-20 08:49:28 +09:00
Chris Lu	b938df97a2	remove unused parameter	2021-07-19 02:59:12 -07:00
Chris Lu	450222dd64	add remote to filer.Entry and filer_pb entry, add RemoteConf	2021-07-19 02:47:27 -07:00
Chris Lu	8dc5def435	proto: add remote	2021-07-19 01:12:31 -07:00
Chris Lu	99155e4880	add `-retry` option for `volume.fix.replication` fix https://github.com/chrislusf/seaweedfs/issues/2191	2021-07-16 12:13:46 -07:00
Chris Lu	2f209675ab	Added `-retry` option for `volumeServer.evacuate` related to https://github.com/chrislusf/seaweedfs/issues/2191	2021-07-16 12:08:21 -07:00
Chris Lu	fb7a1be1c4	refactor	2021-07-16 11:47:37 -07:00
nivekuil	2faf96f002	cassandra: Use LocalOne instead of One consistency	2021-07-16 04:29:46 -07:00
Chris Lu	4be5b4ff5c	[volume.check.disk] skip errors and check next volumes fix https://github.com/chrislusf/seaweedfs/issues/2129	2021-07-16 02:03:32 -07:00
Chris Lu	e95166d739	S	2021-07-15 17:43:38 -07:00
Chris Lu	a45bbc0b75	2.59	2021-07-15 15:52:22 -07:00
Chris Lu	f0042f62dd	readable logs	2021-07-15 12:17:48 -07:00
Chris Lu	18c40686d9	s3: multipart upload miss data if file is chunked in 4MB fix https://github.com/chrislusf/seaweedfs/issues/2195	2021-07-15 11:56:28 -07:00
Chris Lu	5a838dbe53	fix compilation	2021-07-13 11:40:21 -07:00
Chris Lu	6103649ffb	shell: volume.check.disk adds retries in case the volumes are just moved related to https://github.com/chrislusf/seaweedfs/issues/2194	2021-07-13 11:19:56 -07:00
Chris Lu	49c66e88a0	volume: change all writes to fsync during graceful stopping fix https://github.com/chrislusf/seaweedfs/issues/2193	2021-07-13 01:29:57 -07:00
Chris Lu	01adc567aa	shell: volume.fsck deletes a volume is the volume has only orphaned data. fix https://github.com/chrislusf/seaweedfs/issues/2190	2021-07-12 11:22:00 -07:00
Chris Lu	297b41266b	2.58	2021-07-12 01:33:47 -07:00
Chris Lu	b194f91f47	add version to filer configuration response	2021-07-12 01:23:20 -07:00
bingoohuang	9e48bff8ee	fix typo in fs.configure help message text	2021-07-12 13:51:21 +08:00
Chris Lu	d013d6d968	shell: volume.fsck "reallyDeleteFromVolume" should send padded file ids to delete fix https://github.com/chrislusf/seaweedfs/issues/2188	2021-07-10 23:16:06 -07:00
Chris Lu	3d624d1e16	rename	2021-07-09 03:19:21 -07:00
Chris Lu	ecce300964	s3 config read via grpc	2021-07-09 02:48:03 -07:00
Chris Lu	da7bd62822	/etc files are stored inside metadata store	2021-07-09 02:33:14 -07:00
Chris Lu	a6d73e0a66	Merge pull request #2185 from bingoohuang/master show RemoteVolumes/EcVolumes only if it is not empty	2021-07-06 00:28:07 -07:00
bingoohuang	ed57a55eae	show RemoteVolumes/EcVolumes only if it is not empty	2021-07-06 15:20:18 +08:00
Chris Lu	44b50b2fdf	another fix related to 64 bit alignment fix https://github.com/chrislusf/seaweedfs/issues/2177	2021-07-05 16:30:43 -07:00
Chris Lu	141388367e	Merge pull request #2183 from bingoohuang/master extract embed html of master/volume/filer ui to separate files	2021-07-05 03:22:26 -07:00
bingoohuang	44a2538f67	extract embed html of master/volume/filer ui to separate files	2021-07-05 18:09:44 +08:00
Chris Lu	e2da647fa3	Merge pull request #2182 from qieqieplus/fix-event-notification Subscribe meta data: sync empty notification with timestamp	2021-07-05 02:00:53 -07:00
qieqieplus	233103f6b2	sync empty notification with timestamp	2021-07-05 16:01:16 +08:00
bingoohuang	5dbbe19c8b	extract embed toml example to separate files	2021-07-05 11:16:49 +08:00
Chris Lu	5bcc77b46c	volume: default readMode to proxy	2021-07-03 15:55:56 -07:00
Chris Lu	f5fa0b08fd	2.57	2021-07-03 15:10:57 -07:00
Chris Lu	a024254ad7	logging	2021-07-03 14:51:01 -07:00
Chris Lu	d39b2689a5	S3 authorization: StreamingSigned enforces access control fix https://github.com/chrislusf/seaweedfs/issues/2180	2021-07-03 14:50:53 -07:00
Chris Lu	fa0dab6029	mount: rename also recursively move file handles related to https://github.com/chrislusf/seaweedfs/issues/2169	2021-07-03 02:59:35 -07:00
Chris Lu	8fe75692ee	volume: address "unaligned 64-bit atomic operation" fix https://github.com/chrislusf/seaweedfs/issues/2177	2021-07-02 13:57:43 -07:00
Chris Lu	2cb8b31ea7	Merge pull request #2179 from nivekuil/tokenpolicy Cassandra: Use TokenAwareHostPolicy with fallback to localDC by default	2021-07-02 13:51:53 -07:00
nivekuil	8425705643	Cassandra: Use TokenAwareHostPolicy by default with fallback See https://pkg.go.dev/github.com/gocql/gocql#hdr-Data_center_awareness_and_query_routing	2021-07-02 13:50:01 -07:00
Chris Lu	d8bda0b229	locate the weed binary	2021-07-02 13:33:17 -07:00
danielflira	586e066897	fix weed fuse parameters parsing	2021-07-02 16:52:52 -03:00
lyg	1a4db87e19	fix s3 metadata error with multipart upload	2021-07-02 11:00:42 +08:00
Chris Lu	c6d4c16079	S3: add metadata with multipart upload fix https://github.com/chrislusf/seaweedfs/issues/2173	2021-07-01 19:12:11 -07:00
Chris Lu	2420c60fc4	log reading adds delay between retries	2021-07-01 14:01:25 -07:00
Chris Lu	067eb15e70	remove debug messages	2021-07-01 01:24:07 -07:00
Chris Lu	b624090398	go fmt	2021-07-01 01:21:14 -07:00
Chris Lu	215b169562	mount: recursively rename locally	2021-07-01 01:19:31 -07:00
Chris Lu	c795183887	return node itself as directory handler	2021-07-01 01:19:31 -07:00
Chris Lu	30dbe98dff	Update Makefile	2021-07-01 01:19:31 -07:00
Chris Lu	07f20155fd	small optimization	2021-07-01 01:19:31 -07:00
danielflira	a66112c9d2	configurable fusermount path	2021-07-01 00:07:54 -03:00
Kevin Liu	253b4ed898	remote -> proxy in server to match volume flags noticed this was missed in https://github.com/chrislusf/seaweedfs/pull/2168	2021-06-30 13:05:45 -07:00
zhangsong	20d33ae025	add proxy mode to read non-local volumes	2021-06-30 18:33:18 +08:00
zhangsong	7566782c2e	add proxy mode to read non-local volumes	2021-06-30 17:28:37 +08:00
陈杨文	6f683e6572	also failed on mips64 alpine support mips64, will try to build on next version	2021-06-30 04:36:00 +08:00
Chris Lu	41db292332	skip s390 ppc64le due to https://gitlab.alpinelinux.org/alpine/aports/-/merge_requests/22703	2021-06-29 10:06:38 -07:00
Chris Lu	3668d10664	range query for mp4 video play fix https://github.com/chrislusf/seaweedfs/issues/2156	2021-06-29 02:13:29 -07:00
Chris Lu	24e11d1e90	look back when adding to sorted values look back when adding to sorted values, before adding it to overflow	2021-06-28 22:46:49 -07:00
Chris Lu	fc8dd58aea	volume: large_volume version has bug when using in memory index fix https://github.com/chrislusf/seaweedfs/issues/2162	2021-06-28 15:48:07 -07:00
Chris Lu	a2979aa051	2.56	2021-06-27 23:33:45 -07:00
Chris Lu	08377fecb8	fixes	2021-06-27 23:32:57 -07:00
Chris Lu	17477b37d5	sleep before re-reading the messages If there are no more metadata changes and the client disconnects, it would go into a busy loop without this fix.	2021-06-27 06:31:04 -07:00
Chris Lu	c764596f96	filer: slow metadata topic read may lose some change events fix https://github.com/chrislusf/seaweedfs/issues/2117	2021-06-27 05:54:16 -07:00
Chris Lu	cc7714fdbe	logging changes to debug	2021-06-27 05:54:16 -07:00
thephoenixofthevoid	8d70ba2eaa	Fix: Loosing environment variables at StartProcess	2021-06-26 15:20:27 +03:00
Chris Lu	1cac2f2278	shell: volume.fsck checks missing chunks in filer fix https://github.com/chrislusf/seaweedfs/issues/2154	2021-06-24 23:56:24 -07:00
Chris Lu	9dd09bbb33	refactor	2021-06-24 17:22:53 -07:00
Chris Lu	c2e0a75c1f	adjust logs	2021-06-24 12:46:00 -07:00
Chris Lu	78b1fb921c	adjust log level	2021-06-23 20:59:54 -07:00
Chris Lu	05af54ad10	2.55 trigger migrated travis build	2021-06-23 00:41:04 -07:00
Chris Lu	d4b0dcf442	Merge pull request #2147 from jonaz/process_metrics Add process metrics of weed itself	2021-06-22 04:13:17 -07:00
Jonas Falck	829b195084	Add process metrics of weed itself	2021-06-22 13:09:42 +02:00
Chris Lu	d474ce6fe3	master: avoid repeated leader redirection fix https://github.com/chrislusf/seaweedfs/issues/2146	2021-06-21 22:56:07 -07:00
danielflira	f1d207a0fe	start weed with mount in background	2021-06-20 02:48:46 -03:00
Chris Lu	b3eb4fecc7	2.54	2021-06-19 03:48:15 -07:00
Chris Lu	f24bb9e688	mount: fix for deletion stopped working since 2.53 fix https://github.com/chrislusf/seaweedfs/issues/2138 due to `4d0cbd2700`	2021-06-19 03:46:39 -07:00
Chris Lu	88d52adfdd	remove unused fields	2021-06-18 15:35:22 -07:00
Chris Lu	56eb522b13	fix stats when a collection is deleted	2021-06-15 21:11:31 -07:00
Chris Lu	055374a50b	FUSE: skip flushing if file is deleted related to https://github.com/chrislusf/seaweedfs/issues/2110	2021-06-15 12:45:23 -07:00
Konstantin Lebedev	6aa1a56ec8	avoid crashes Galera Cluster https://github.com/chrislusf/seaweedfs/issues/2125	2021-06-15 18:12:39 +05:00
Chris Lu	ebe971da2e	Merge pull request #2127 from nagy/check-history-error Check for history file errors as well	2021-06-15 03:29:21 -07:00
Chris Lu	7a81caa31e	2.53	2021-06-13 17:12:34 -07:00
Chris Lu	4d0cbd2700	skip cookie checking if from grpc api	2021-06-13 16:16:11 -07:00
Chris Lu	7225cb4ac5	add block and mutex profiling	2021-06-13 16:15:54 -07:00
Chris Lu	28a4a1f8d6	fix for mysql2 postgres2 on fast dropping buckets	2021-06-13 07:31:56 -07:00
Chris Lu	1e76fc994a	filer.copy: zero fileSize for directories	2021-06-13 07:09:06 -07:00
Chris Lu	ed6aa13520	minor	2021-06-12 18:36:25 -07:00
Chris Lu	ee6c67682c	minor	2021-06-12 02:52:41 -07:00
Chris Lu	9357911a95	remove all bucket metadata fix https://github.com/chrislusf/seaweedfs/issues/2118	2021-06-10 23:37:54 -07:00
Chris Lu	b71c3cfba4	avoid possible empty object	2021-06-10 22:17:53 -07:00
Chris Lu	310e31424e	adjust the error output fix https://github.com/chrislusf/seaweedfs/issues/2123	2021-06-10 21:55:13 -07:00
Chris Lu	8b382a8209	refactor	2021-06-10 21:50:21 -07:00
Chris Lu	33b87244ef	refactoring	2021-06-10 16:54:36 -07:00
Daniel Nagy	7ce3bee0c0	Check for history file errors as well	2021-06-08 11:14:49 +02:00
Daniel Nagy	0e3adde47f	Return correct exitcode when `wheed upload` fails	2021-06-07 23:38:14 +02:00
Chris Lu	76f24af79a	2.52	2021-06-07 12:13:23 -07:00
Chris Lu	5e6dfbc25f	locks for data racing	2021-06-07 12:04:50 -07:00
Chris Lu	452c6ef183	limits concurrent uploads for one file	2021-06-06 23:05:17 -07:00
Chris Lu	8295e2feb6	skip md5 checking for now because of race condition	2021-06-06 22:16:32 -07:00
Chris Lu	c8dea3dd89	2.51	2021-06-06 21:54:00 -07:00
Chris Lu	19caeb7b02	fix writing the small file	2021-06-06 20:57:03 -07:00
Chris Lu	bb45dea15a	filer: parallel data upload	2021-06-06 20:23:36 -07:00
Chris Lu	e00443a940	mount: adjust starting order avoid possible nil wfs.Server	2021-06-06 20:22:42 -07:00
Chris Lu	44f1ba6894	refactor	2021-06-06 18:43:04 -07:00
Chris Lu	6c82326575	use bytes.Buffer to reduce memory allocation and gc	2021-06-06 13:42:36 -07:00
Chris Lu	9cba5cca0b	optionally disable concurrent upload limit	2021-06-06 13:13:33 -07:00
Chris Lu	21ad9a4ac2	filer: mongodb avoids E11000 duplicate key error collection: seaweedfs.filemeta index: directory_1_name_1 dup key	2021-06-06 13:12:01 -07:00
Chris Lu	e6ba2f9c37	verbose but cleaner logic to invalidate fuse cache	2021-06-05 13:09:37 -07:00
Chris Lu	fadc1febdb	FUSE: invalidate FUSE cached entries fix https://github.com/chrislusf/seaweedfs/issues/2108	2021-06-05 02:23:07 -07:00
Chris Lu	ab606dec2a	filer: add path-specific option to enforce readonly	2021-06-04 01:03:41 -07:00
Chris Lu	0a5388744c	Merge pull request #2105 from Woellchen/current_and_parent_dot_directories Return artificial . and .. directories	2021-06-02 12:54:44 -07:00
Patrick Schmidt	77100754e6	Return artificial . and .. directories	2021-06-02 21:28:02 +02:00
Chris Lu	62142ff1d2	Merge pull request #2104 from kmlebedev/sync_volume_on_close avoid data loss after restarting a container with a volum server	2021-06-02 11:47:57 -07:00
Chris Lu	cc34475012	remove file handle locking for setattr	2021-06-02 11:44:12 -07:00
Konstantin Lebedev	99ef280c7c	avoid data loss after restarting a container with a volum server	2021-06-02 17:07:19 +05:00
Chris Lu	3db1642392	mount: skip persisting metadata if file is open	2021-06-02 03:40:08 -07:00
Chris Lu	556cc3a4ca	mount: avoid exception if disk cache is not initialized related to https://github.com/chrislusf/seaweedfs/issues/2102	2021-05-31 16:42:55 -07:00
Chris Lu	615cb24ba6	shell: use tab print style for completion	2021-05-31 03:29:29 -07:00
Chris Lu	c5de97ddbe	keep alive for streaming connections fix https://github.com/chrislusf/seaweedfs/issues/2096 related to https://github.com/chrislusf/seaweedfs/issues/1277	2021-05-31 01:26:24 -07:00
Chris Lu	2b60e2abb1	only disallow streaming signed when no auth enabled fix https://github.com/chrislusf/seaweedfs/issues/2101	2021-05-31 01:03:04 -07:00
Chris Lu	4233ad3f07	2.50	2021-05-30 20:40:30 -07:00
Chris Lu	1456616a77	recreate grpc connections if too many errors address https://github.com/chrislusf/seaweedfs/issues/2098	2021-05-30 00:07:43 -07:00
Chris Lu	fb8036385a	s3: save metadata during put-object fix https://github.com/chrislusf/seaweedfs/issues/2092	2021-05-29 14:14:30 -07:00
Chris Lu	2d7b4e5bb6	filer.backup: escape colon from path on windows fix https://github.com/chrislusf/seaweedfs/issues/2084	2021-05-29 06:45:27 -07:00
Chris Lu	ae185b997f	Merge pull request #2099 from danielflira/mount-helper fix parameter multiple values	2021-05-28 21:57:15 -07:00
danielflira	849f36c1ac	fix parameter multiple values	2021-05-29 00:37:25 -03:00
Eugeniy Kozlov	6e8bd16819	freespace params fix	2021-05-28 14:19:24 +03:00
Chris Lu	4d55132c7d	scaffold add comments	2021-05-27 14:40:25 -07:00
Chris Lu	45bffc92a8	filer.backup: fix cloud sinks when updating entry	2021-05-26 14:53:11 -07:00
Chris Lu	921e0d5008	remove verbose log	2021-05-26 14:43:34 -07:00
Chris Lu	1a70cb9b63	local sink: write and update files fix issues with https://github.com/chrislusf/seaweedfs/issues/2084	2021-05-26 14:42:21 -07:00
Chris Lu	e699d16c85	Merge pull request #2085 from danielflira/mount-helper create fuse subcommand to use weed with mount	2021-05-26 12:19:40 -07:00
danielflira	84488ebb33	replace filer.remote and parts[1]	2021-05-26 12:07:36 -03:00
Chris Lu	ac9bf71544	use jquery 3.6.0	2021-05-26 00:21:11 -07:00
danielflira	877c192af2	create fuse subcommand to use weed with mount	2021-05-25 23:32:35 -03:00
Chris Lu	67dd094b35	sqlite: does not support non-linux/darwin/windows	2021-05-25 17:36:15 -07:00
Chris Lu	dce1f02c9e	filer.backup: backup to local directory optionally is incremental fixed one issue with https://github.com/chrislusf/seaweedfs/issues/2084	2021-05-25 17:19:20 -07:00
Konstantin Lebedev	b612d5aebd	s3 test get w/ If-Match: bogus ETag	2021-05-24 16:59:44 +05:00
Konstantin Lebedev	69c768870b	- object write cache control - object write expires	2021-05-24 15:43:55 +05:00
Konstantin Lebedev	84dce32a57	Merge branch 'master' into head_check_all_chunks	2021-05-24 12:28:19 +05:00
Konstantin Lebedev	0e404cd84b	Chunk download stats	2021-05-24 12:14:50 +05:00
Konstantin Lebedev	bb42633522	rm chunk Download stats	2021-05-24 12:08:12 +05:00
Chris Lu	064269bb57	filer: Support sqlite as filer meta store	2021-05-23 23:58:28 -07:00
Chris Lu	b70aeb7585	bucket list only directories	2021-05-23 10:36:22 -07:00
Chris Lu	42fb03a66e	2.49	2021-05-23 00:51:47 -07:00
Chris Lu	3d3fa43542	filer: re-create grpc connections if having transport error fix https://github.com/chrislusf/seaweedfs/issues/2070	2021-05-22 17:46:53 -07:00
Chris Lu	2d0a0733ec	filer: bootstrap from peer filer fix https://github.com/chrislusf/seaweedfs/issues/1861	2021-05-22 03:17:02 -07:00
Chris Lu	e5a2bf1287	s3: deprecating filer.options.buckets_fsync	2021-05-22 00:24:23 -07:00
Chris Lu	431684798b	s3: add errors if requests are signed by no authentication is setup fix https://github.com/chrislusf/seaweedfs/issues/2075	2021-05-21 14:08:47 -07:00
Chris Lu	2f136a04a1	organize a bit better for temp file directory	2021-05-21 13:02:18 -07:00
Konstantin Lebedev	3325b850cc	rm func CheckAllChunkViews	2021-05-21 16:05:59 +05:00
Konstantin Lebedev	26a4f34a57	del checks url err logging stats chunk fetch	2021-05-21 15:59:12 +05:00
Chris Lu	e8b7d4ad64	simplify a bit	2021-05-21 02:10:44 -07:00
Chris Lu	0b00edfc3b	waits in case master lost connection	2021-05-21 02:09:22 -07:00
Chris Lu	f37a4cbd0f	randomize initial filer	2021-05-21 01:41:34 -07:00
Chris Lu	5d77840cff	adjust help message	2021-05-21 01:38:57 -07:00
Chris Lu	dc1309f084	FUSE mount: support multiple filers fix https://github.com/chrislusf/seaweedfs/issues/2015 fix https://github.com/chrislusf/seaweedfs/issues/1531	2021-05-21 01:28:00 -07:00
Konstantin Lebedev	42f631f549	fix https://github.com/chrislusf/seaweedfs/issues/2046	2021-05-20 13:19:35 +05:00
Chris Lu	30c67e3652	minor	2021-05-19 23:59:39 -07:00
Konstantin Lebedev	03d1199d5f	Revert "revert PR #1903 avoid http error: superfluous response.WriteHeader" This reverts commit `ac71117e`	2021-05-20 11:45:21 +05:00
Chris Lu	87a32bfef4	avoid possible nil when node is disconnected from its parent fix https://github.com/chrislusf/seaweedfs/issues/2073	2021-05-19 10:02:01 -07:00
Chris Lu	1aa7e99a89	skip file not found error when deleting	2021-05-15 09:37:39 -07:00
Chris Lu	45a762223a	2.48	2021-05-14 10:26:42 -07:00
Chris Lu	789465d99e	set renamed item to new directory related to https://github.com/chrislusf/seaweedfs/issues/2064	2021-05-13 22:52:19 -07:00
Chris Lu	83c037e093	fix logs	2021-05-13 22:41:19 -07:00
Chris Lu	5a29d284fa	ensure proper directory name a fix related to https://github.com/chrislusf/seaweedfs/issues/2064	2021-05-13 10:18:35 -07:00
Chris Lu	3ff307e842	reduce logs	2021-05-13 00:32:37 -07:00
Chris Lu	a48ebd7c73	mount: read file when file is still being written a possible fix for https://github.com/chrislusf/seaweedfs/issues/2065	2021-05-13 00:32:23 -07:00
Chris Lu	26a55bbb5c	Adjust error message when bucket name conflicts with existing collections fix https://github.com/chrislusf/seaweedfs/issues/2069	2021-05-12 22:30:39 -07:00
Chris Lu	ceb620a30a	directory rename: change directory name after renaming fix https://github.com/chrislusf/seaweedfs/issues/2068	2021-05-12 22:04:47 -07:00
Chris Lu	b430d1b6ee	filer.copy: "check.size" before copying files fix https://github.com/chrislusf/seaweedfs/issues/2067	2021-05-12 21:45:39 -07:00
Chris Lu	0f7b43af99	2.47	2021-05-11 10:12:15 -07:00
Chris Lu	d2d36a3f9d	master: avoid creating too many volumes fix https://github.com/chrislusf/seaweedfs/issues/2062	2021-05-11 10:05:31 -07:00
Chris Lu	4596e64710	2.46	2021-05-10 21:58:37 -07:00
Chris Lu	e55aa41690	Merge branch 'upload_via_temp_file'	2021-05-10 21:48:18 -07:00
Chris Lu	1737af480a	adjust logs	2021-05-10 21:47:51 -07:00
Chris Lu	d06ecc2649	working properly	2021-05-10 21:47:07 -07:00
Chris Lu	fe2edd4b50	2.45	2021-05-10 13:53:03 -07:00
Chris Lu	55d547afec	switch back to in memory uploading for now	2021-05-10 13:50:10 -07:00
Chris Lu	9a6aa00e9d	avoid nil locations fix https://github.com/chrislusf/seaweedfs/issues/2059	2021-05-10 02:39:52 -07:00
Chris Lu	da0a4e775b	Revert "Revert "Merge pull request #2027 from bingoohuang/master"" This reverts commit `d74cdf0115`.	2021-05-10 00:13:55 -07:00
Chris Lu	aa6949ef25	2.44	2021-05-09 23:28:48 -07:00
Chris Lu	ba77833343	EOF handling	2021-05-09 23:04:24 -07:00
Chris Lu	f4cf30b497	mount: write to tmp file before uploading	2021-05-09 22:56:10 -07:00
Chris Lu	280e7cf289	mount: in case the set attribute is called before persisting the file	2021-05-09 22:55:30 -07:00
Chris Lu	74052064b6	add one missing optimization	2021-05-09 17:25:42 -07:00
Chris Lu	59ace54925	refactor	2021-05-09 17:22:30 -07:00
Chris Lu	50be19d23e	refactor	2021-05-09 15:33:01 -07:00
Chris Lu	55e060cf61	refactor	2021-05-09 15:28:54 -07:00
Chris Lu	735e65be17	refactor	2021-05-09 15:22:38 -07:00
Chris Lu	93e84a12f2	refactor	2021-05-09 15:15:18 -07:00
Chris Lu	3942e3b2ef	a better fix	2021-05-09 01:42:19 -07:00
Chris Lu	957e1a1bc1	fuse: important: if filer -filer.path is not root, directory listing will fail	2021-05-09 00:32:21 -07:00
Chris Lu	7ca75347ec	minor	2021-05-07 21:56:45 -07:00
Chris Lu	8f8738867f	add retry to assign volume fix https://github.com/chrislusf/seaweedfs/issues/2056	2021-05-07 07:29:26 -07:00
Chris Lu	007401f3a0	remove duplicated code	2021-05-07 07:14:24 -07:00
Chris Lu	aaad4b578b	Merge pull request #2049 from qieqieplus/async-assign ahead of time volume assignment	2021-05-06 10:00:25 -07:00
qieqieplus	c4d32f6937	ahead of time volume assignment	2021-05-06 18:55:44 +08:00
Chris Lu	55a8f57381	go fmt	2021-05-06 03:37:51 -07:00
Chris Lu	5753749c90	remove verbose logs	2021-05-06 03:34:34 -07:00
Chris Lu	38f411219a	mount: skip local chunk cache if opened write only	2021-05-06 03:31:40 -07:00
Chris Lu	c899bdf063	a little optimization	2021-05-06 03:03:00 -07:00
Chris Lu	3eb336e0b0	report error only for the first multipart upload the glog.Errorf would always print for s3 multipart uploads	2021-05-06 01:57:54 -07:00
Chris Lu	76c48ffe27	optional parallel copy ec shards fix https://github.com/chrislusf/seaweedfs/issues/2048	2021-05-06 01:53:35 -07:00
Chris Lu	ac71117ee6	revert PR #1903 avoid http error: superfluous response.WriteHeader	2021-05-05 15:11:39 -07:00
Chris Lu	24efa31e49	Merge pull request #2045 from qieqieplus/fix-vacuum-commit	2021-05-05 07:41:38 -07:00
qieqieplus	ac26080bd2	fix concurrent vacuum & delete panic	2021-05-05 17:54:50 +08:00
Chris Lu	e24ba2aadc	filer: delete specific tags fix https://github.com/chrislusf/seaweedfs/issues/2041	2021-05-02 21:53:43 -07:00
Chris Lu	e87e6ef33c	s3: return 404 if bucket does not exist fix https://github.com/chrislusf/seaweedfs/issues/2039	2021-05-02 21:30:37 -07:00
Chris Lu	a4bb37a5fe	add debug info	2021-05-01 16:10:26 -07:00
Chris Lu	2e56407c6b	fix visited checking	2021-05-01 16:09:29 -07:00
Chris Lu	c48ef78670	2.43	2021-05-01 00:39:04 -07:00
Chris Lu	3a86d4dbfd	mount: fix directory invalidation fix https://github.com/chrislusf/seaweedfs/issues/2038	2021-04-30 22:51:06 -07:00
Chris Lu	d74cdf0115	Revert "Merge pull request #2027 from bingoohuang/master" Need to revert because docker image build failed. The docker apk package only has go 1.15.	2021-04-30 03:36:15 -07:00
Chris Lu	84312e6799	2.42	2021-04-30 03:14:07 -07:00
Chris Lu	fa0d973113	Merge pull request #2037 from utsl42/reader_at make reader_at handle random reads more efficiently for FUSE	2021-04-28 17:31:46 -07:00
Nathan Hawkins	042de9359c	make reader_at handle random reads more efficiently for FUSE	2021-04-28 19:13:37 -04:00
Chris Lu	a26a37dfa3	fix compilation fix related to #2032	2021-04-28 13:36:53 -07:00
Chris Lu	ef94ff6837	Merge pull request #2035 from kmlebedev/fix_chunks_etag fix aws style Etag for chunks	2021-04-28 10:37:18 -07:00
Chris Lu	8ae3ea4b1d	Merge pull request #2034 from kmlebedev/iam_listkeys_filter iam ListAccessKeys filtred by username	2021-04-28 10:34:40 -07:00
Chris Lu	9dca75aea8	Merge pull request #2032 from tobiasmuehl/patch-2 Detect rar archives by mime type	2021-04-28 10:31:19 -07:00
Konstantin Lebedev	c2269123d3	fix aws style Etag for chunks	2021-04-28 22:28:05 +05:00
Konstantin Lebedev	39f636c682	iam ListAccessKeys filtred by username	2021-04-28 17:15:22 +05:00
Tobias Mühl	a8864e2abd	Detect rar archives by mime type RAR archives might not have .rar extension, see [Wikipedia](https://en.wikipedia.org/wiki/RAR_(file_format))	2021-04-28 15:54:19 +07:00
Tobias Mühl	12a7e87007	Do not compress brotli archives	2021-04-28 15:51:49 +07:00
Konstantin Lebedev	a48785c7df	auth use bucket wild cards	2021-04-27 21:45:40 +05:00
bingoohuang	7a9d27fce8	promote to go:embed instead of github.com/rakyll/statik	2021-04-27 17:22:24 +08:00
bingoohuang	cf552417a7	minFreeSpace refactored	2021-04-27 10:37:24 +08:00
bingoohuang	31f1cdeac2	minFreeSpace argument allows size like 10GiB	2021-04-26 18:48:34 +08:00
Chris Lu	86185262bb	2.41	2021-04-24 16:54:36 -07:00
Chris Lu	79f2e780c1	ensure name pattern checking is case sensitive	2021-04-24 11:51:23 -07:00
Chris Lu	ddc8643ee0	filer: directory listing adds namePatternExclude fix https://github.com/chrislusf/seaweedfs/issues/2023	2021-04-24 11:49:03 -07:00
Chris Lu	f0ad172e80	shell: show which server holds the lock fix https://github.com/chrislusf/seaweedfs/issues/1983	2021-04-22 23:56:35 -07:00
Chris Lu	89eb9f6e70	clean up .uploads directory in mysql and postgres tables fix https://github.com/chrislusf/seaweedfs/issues/1957 When no uploads are running, you can run this SQL to clean up. delete from <bucket_name> where directory like '/.uploads/%'	2021-04-22 23:23:23 -07:00
Chris Lu	46ef1811a1	correct help message	2021-04-22 22:26:38 -07:00
Chris Lu	11c120c040	master UI adds volume size limit	2021-04-22 14:22:48 -07:00
Chris Lu	6e5df901e4	adjust package names	2021-04-22 14:22:48 -07:00
Chris Lu	ca998328c2	do not add new volumes when below minFreeSpacePercent fix https://github.com/chrislusf/seaweedfs/issues/2017	2021-04-21 23:11:11 -07:00
Chris Lu	cd7bf1a72a	filer.copy copy empty folders fix https://github.com/chrislusf/seaweedfs/issues/2016	2021-04-21 11:17:43 -07:00
Chris Lu	a8114da02d	avoid thundering herd effect transient errors may cause thundering herd effect to all trying to recover from remove ec shards	2021-04-21 10:17:12 -07:00
Chris Lu	ae74d8f02a	fix error message related to https://github.com/chrislusf/seaweedfs/issues/2012	2021-04-21 01:40:16 -07:00
Chris Lu	11c405fc85	ensure file handles are released	2021-04-20 19:56:55 -07:00
liuxiaobo	c31c5e829c	fix path-specific filer store comment error	2021-04-20 10:08:58 +08:00
Chris Lu	83cf94ad2d	delay new file creation unless file is opened exclusively	2021-04-19 10:58:25 -07:00
Chris Lu	e983f91b03	2.40	2021-04-18 13:58:01 -07:00
Chris Lu	d1c813c470	let the fuse library manage directory id otherwise, on mac, during large directory deletion, if some ReaDirAll happens, the lib seems confused about the directories, and some child directories are not deleted.	2021-04-18 13:07:28 -07:00
Chris Lu	372872ebbf	set root node inode number	2021-04-18 13:07:28 -07:00
Chris Lu	e332da4837	set inode value	2021-04-18 13:07:28 -07:00
Chris Lu	2acf6be24e	resend the http request if connection is stale	2021-04-18 13:07:28 -07:00
Chris Lu	6cbd786db9	correctly runs git clone	2021-04-18 13:07:28 -07:00
Chris Lu	d41e6826d3	adjust logging	2021-04-18 13:06:38 -07:00
Chris Lu	d9a2a7f1c4	WIP no memory issue if some directory is removed, it may have this error $ rm -Rf ~/tmp/m2/s1 rm: fts_read: Device not configured	2021-04-18 13:06:38 -07:00
Chris Lu	54410ca955	cleaner way to set readonly	2021-04-18 10:02:02 -07:00
Konstantin Lebedev	198688c717	revert volume etag	2021-04-16 23:22:31 +05:00
Chris Lu	c83ab91e2e	remove unused variable	2021-04-16 10:34:02 -07:00
Chris Lu	3074e9b428	ensure consistent inode value	2021-04-15 22:42:24 -07:00
Chris Lu	b971317a16	avoid possible corrupted file names	2021-04-15 11:41:34 -07:00
Chris Lu	283d703d50	adjust text	2021-04-15 11:29:58 -07:00
Chris Lu	609e228578	avoid forward slash in file names	2021-04-15 10:53:04 -07:00
Chris Lu	ba92f2e714	add node.selectedVolumes fix https://github.com/chrislusf/seaweedfs/issues/1990	2021-04-15 03:19:28 -07:00
Chris Lu	16c0304416	ensure to delete on filer also	2021-04-15 02:29:04 -07:00
Chris Lu	217e0f9066	mount: remove folder recursively	2021-04-15 01:51:10 -07:00
Chris Lu	3e669e6d7b	mostly refactoring, add some error handling	2021-04-14 23:33:37 -07:00
Chris Lu	07f712c83f	fix typo	2021-04-14 23:21:38 -07:00
Chris Lu	36c79de3f4	fuse mount: dir ReadDirAll avoid extra conversion to filer_pb.Entry	2021-04-14 23:21:24 -07:00
Chris Lu	e41766feb6	fuse mount: dir lookup avoids extra conversion to filer_pb.Entry object	2021-04-14 22:38:34 -07:00
Chris Lu	1adc8f86ea	lighten up File object file.entry only exists when file.isOpen	2021-04-14 20:49:15 -07:00
Chris Lu	6bc09b18c4	truncate is a bit faster to reuse the storage	2021-04-14 20:26:56 -07:00
Chris Lu	c04b7e106f	mount: remove entry from Dir object	2021-04-14 20:26:13 -07:00
Chris Lu	fc0cbf565f	add option to obfuscate the file names	2021-04-14 15:37:24 -07:00
Chris Lu	e75633c64f	volume.check.disk: break loop for read only volumes fix https://github.com/chrislusf/seaweedfs/issues/2002	2021-04-14 12:40:13 -07:00
Chris Lu	9d50867d08	volume.tier.move: avoid data loss when destination volume server already has the volume fix https://github.com/chrislusf/seaweedfs/issues/2001	2021-04-14 10:26:26 -07:00
Chris Lu	ff4c1d5965	adjust logging fix https://github.com/chrislusf/seaweedfs/issues/1999	2021-04-14 10:04:26 -07:00
qieqieplus	270645f8d7	fix #1996	2021-04-14 18:29:28 +08:00
Chris Lu	90677e1097	ensure to call line.Close() fix https://github.com/chrislusf/seaweedfs/issues/1995 similar to https://github.com/peterh/liner/issues/104	2021-04-14 01:33:21 -07:00
Chris Lu	3f3268cd1b	go fmt	2021-04-14 00:30:16 -07:00
Chris Lu	ca0f07a188	move file reader, entryViewCache to file handle reduce file object size	2021-04-14 00:29:58 -07:00
Chris Lu	5985a7d38d	add log level possible values fix https://github.com/chrislusf/seaweedfs/issues/1989	2021-04-12 21:15:51 -07:00
Chris Lu	f5de42fae3	Merge pull request #1975 from kmlebedev/iam_handlers IAM handlers	2021-04-12 12:07:45 -07:00
Chris Lu	1e033d45b8	simpler logic related to https://github.com/chrislusf/seaweedfs/pull/1981	2021-04-12 12:04:53 -07:00
Chris Lu	52200a903b	Merge pull request #1981 from ueni-ltd/if-modified-since Fix If-Modified-Since behavior	2021-04-12 12:02:54 -07:00
Chris Lu	519b0e1e49	filer: upload to a directory without "/" suffix fix https://github.com/chrislusf/seaweedfs/issues/1988	2021-04-12 11:56:56 -07:00
Konstantin Lebedev	8e02e138ea	Merge branch 'upstreamMaster' into iam_handlers	2021-04-12 11:22:51 +05:00
Chris Lu	742ab1ec81	2.39	2021-04-11 19:47:11 -07:00
Chris Lu	0df5b53ad8	adjust help message	2021-04-11 00:26:28 -07:00
Chris Lu	f62c153274	go fmt	2021-04-10 23:48:18 -07:00
Chris Lu	af313dff58	add gateway for easier POST and DELETE blobs	2021-04-10 23:47:47 -07:00
Konstantin Lebedev	5021bea698	GetUserPolicy	2021-04-10 23:57:45 +05:00
Chris Lu	0b82edc0d2	filer: avoid stuck uploader fix https://github.com/chrislusf/seaweedfs/issues/1980 reverting the file upload batch executor	2021-04-09 13:05:15 -07:00
Chris Lu	93f4146ffa	properly release the view cache	2021-04-09 12:36:39 -07:00
Merlin Gaillard	f952f979d1	filer: return 304 when If-Modified-Since == Last-Modified	2021-04-09 15:04:17 +02:00
Merlin Gaillard	4d4acc715e	s3api: handle 304 response code from filer	2021-04-09 12:13:19 +02:00
Chris Lu	6deb647a8f	mount: fix possible memory leak if many files are read repeatedly, their metadata are accumulated in memory. This fix cleared the metadata after the file is read.	2021-04-08 19:47:31 -07:00
Konstantin Lebedev	ba175f81b5	add auth aws signV4	2021-04-08 17:40:47 +05:00
Konstantin Lebedev	995ae91007	add DeleteUserPolicy	2021-04-08 11:16:36 +05:00
Philippe Pepiot	42a761ee20	Fix typo in weed filer long help	2021-04-07 22:47:23 +02:00
Chris Lu	3be061994f	skip connection reset error fix https://github.com/chrislusf/seaweedfs/issues/1971 this is because the connections are pooled but the volume server has reset the connection	2021-04-07 00:54:13 -07:00
Chris Lu	c5b08bac1b	remove mac specific mount options	2021-04-07 00:54:13 -07:00
Konstantin Lebedev	f5f8eec8e2	fix get filerGrpcAddress	2021-04-06 13:53:56 +05:00
Konstantin Lebedev	011e6e90ee	Merge branch 'upstreamMaster' into iamapipr	2021-04-06 13:50:33 +05:00
Konstantin Lebedev	ed79baa30f	add tests	2021-04-06 13:43:08 +05:00
Chris Lu	2327c0756b	fix to avoid loop	2021-04-05 23:24:26 -07:00
Chris Lu	a37eca78cd	2.38	2021-04-05 19:41:54 -07:00
Chris Lu	c131764c34	ensure error is sent back in channel fix https://github.com/chrislusf/seaweedfs/issues/1966 avoid shared readErr variable	2021-04-05 19:40:12 -07:00
Chris Lu	0f64f5b9c8	mount: add readOnly option fix https://github.com/chrislusf/seaweedfs/issues/1961	2021-04-04 21:40:58 -07:00
Chris Lu	6eee200c13	2.37	2021-04-04 18:45:48 -07:00
Chris Lu	fbb82a5c9c	skip limiting if limit is zero	2021-04-04 18:38:33 -07:00
Chris Lu	8251d1140e	refactor	2021-04-04 18:38:33 -07:00
Chris Lu	5dca98fe71	fix typo	2021-04-03 11:37:13 -07:00
Chris Lu	5e64f65632	ensure tables are created fix https://github.com/chrislusf/seaweedfs/issues/1957	2021-04-02 12:02:26 -07:00
Chris Lu	319743d330	go fmt	2021-04-02 02:22:59 -07:00
Chris Lu	8e404a1433	go fmt	2021-04-02 02:22:26 -07:00
Chris Lu	bdf2ddddfd	revert to same implementation as before This reverts commit `7e8edc3c4a`.	2021-04-02 02:21:38 -07:00
Chris Lu	67e019d54b	add missing changes	2021-04-02 01:11:44 -07:00
Chris Lu	7e8edc3c4a	refactoring	2021-04-02 01:10:24 -07:00
Chris Lu	cc0df36a9e	smaller file chunks are faster	2021-04-01 02:21:59 -07:00
Chris Lu	cefe66f159	dedicated upload processor avoid thundering effect of overloading volume servers	2021-04-01 02:21:40 -07:00
Chris Lu	1f984d2645	refactor buffer pool	2021-04-01 02:20:00 -07:00
Chris Lu	b5880334fc	refactor	2021-03-30 21:07:34 -07:00
Chris Lu	e79e2ddeed	remove unused MoveEvents	2021-03-30 20:42:44 -07:00
Chris Lu	e06807a55b	adjust logs	2021-03-30 20:36:06 -07:00
Chris Lu	ac875976c0	filer, volume: add concurrent upload size limit to avoid OOM add some back pressure when writes are slow	2021-03-30 02:10:53 -07:00
Chris Lu	a1e18a1384	server add memory profiling	2021-03-30 02:10:53 -07:00
LazyDBA247-Anyvision	9385e2224d	fix postgres2 upsertQuery syntax also	2021-03-30 10:51:52 +03:00
Chris Lu	a5662c9e5e	adjust ordering	2021-03-30 00:26:57 -07:00
Chris Lu	bec74c3e6a	Merge pull request #1948 from LazyDBA247-Anyvision/master add enableUpsert=true	2021-03-30 00:25:01 -07:00
LazyDBA247-Anyvision	9f1cab179c	use a compatible pg syntax for upsert	2021-03-30 10:21:27 +03:00
Chris Lu	a95929e53c	reduce default concurrentWriters to 32	2021-03-30 00:17:52 -07:00
LazyDBA247-Anyvision	7f44d953b5	fix GetBool	2021-03-30 01:36:02 +03:00
LazyDBA247-Anyvision	4c51e6a660	add enableUpsert=true and rename config to upsertQuery	2021-03-30 00:32:03 +03:00
Chris Lu	b61e9f6a1a	adjust ordering	2021-03-29 00:49:50 -07:00
Chris Lu	1ebdbad222	Merge pull request #1946 from LazyDBA247-Anyvision/master Adding Customisation - insertQuery for postgres/mysql	2021-03-29 00:42:48 -07:00
Konstantin Lebedev	8a95f9c10c	iam GetUser	2021-03-29 12:01:44 +05:00
LazyDBA247-Anyvision	96c62bd34d	scaffold - insertQuery Example for upsert query for mysql & postgres variants	2021-03-29 10:01:12 +03:00
LazyDBA247-Anyvision	4a02389eb0	Adding custom insertQuery support for postgres/2 mysql/2	2021-03-29 09:58:13 +03:00
Chris Lu	6b7aa9633f	2.36	2021-03-28 19:09:06 -07:00
Chris Lu	bd7471d877	refactor	2021-03-25 12:05:59 -07:00
李海	06be5dc6c3	log snowflake sequencer nodeid's hex when start, it'll be in part of new assigned fid	2021-03-25 18:59:40 +08:00
李海	69b2dab9c6	add a snowflake sequencer as more robust fid generator, but less compressable than small auto-inc id	2021-03-25 18:49:26 +08:00
Konstantin Lebedev	40938d6a47	SaveInsideFiler S3 Configuration	2021-03-25 12:34:11 +05:00
Konstantin Lebedev	9f26f2815c	SaveAs S3 Configuration	2021-03-25 12:34:11 +05:00
Konstantin Lebedev	82b0463fac	handler PutUserPolicy https://docs.aws.amazon.com/IAM/latest/APIReference/API_PutUserPolicy.html	2021-03-25 12:34:11 +05:00
Konstantin Lebedev	d7719d0542	base handlers	2021-03-25 12:34:11 +05:00
Konstantin Lebedev	03c7953254	init Iam Api Server	2021-03-25 12:34:11 +05:00
Chris Lu	82dfe06066	Merge pull request #1933 from kmlebedev/missingKeysThreshold add missingKeysThreshold for fsck	2021-03-24 10:56:25 -07:00
Konstantin Lebedev	020a5d40c3	avoid counter is null	2021-03-24 22:51:22 +05:00
Konstantin Lebedev	df6cf0a2fa	nonRepairThreshold	2021-03-24 22:07:13 +05:00
Konstantin Lebedev	9ce82a9cd2	GOMAXPROCS use defaults to the value of runtime.NumCPU https://github.com/chrislusf/seaweedfs/issues/1942	2021-03-24 18:45:58 +05:00
Konstantin Lebedev	60972f1c97	rename option repairThreshold	2021-03-24 13:24:49 +05:00
Chris Lu	a801332b0d	filer: return 409 if file conflicts with a directory fix https://github.com/chrislusf/seaweedfs/issues/1938	2021-03-24 01:03:11 -07:00
Chris Lu	c42b95c596	more help message fix https://github.com/chrislusf/seaweedfs/issues/1937	2021-03-23 17:27:57 -07:00
Konstantin Lebedev	40a9e88c07	rename var	2021-03-23 21:42:35 +05:00
Konstantin Lebedev	d21a5bf5d0	add missingKeysThreshold for fsck	2021-03-23 15:04:07 +05:00
Chris Lu	85cff10787	mysql: ensure name column is case sensitive fix https://github.com/chrislusf/seaweedfs/issues/1931	2021-03-23 00:46:50 -07:00
Chris Lu	4abb511db3	make a local copy of the in memory cached data	2021-03-22 22:33:07 -07:00
Chris Lu	3cbc40fa48	avoid creating multiple reader	2021-03-22 22:32:47 -07:00
Chris Lu	2f7c7afdec	refactor: remove unused parameter	2021-03-22 22:13:19 -07:00
Chris Lu	d48dd0c738	use []byte directly instead of bytes.Buffer	2021-03-22 22:12:57 -07:00
Chris Lu	8eb14967ef	shell: volume.check.disk adds optional slow mode fix https://github.com/chrislusf/seaweedfs/issues/1930	2021-03-22 21:01:43 -07:00
Chris Lu	288369cfc7	mount: release resources when Forget() is called address https://github.com/chrislusf/seaweedfs/issues/1929	2021-03-22 16:22:19 -07:00
Chris Lu	5d931eff27	avoid possible nil fix https://github.com/chrislusf/seaweedfs/issues/1928 The nil was because of `dn.Parent().UnlinkChildNode(dn.Id())` in topo.UnRegisterDataNode() function, when the dn leaves the cluster.	2021-03-22 13:24:07 -07:00
Chris Lu	f315eb2bb8	fix build error	2021-03-22 00:12:53 -07:00
Chris Lu	1dd5bc134c	2.35	2021-03-22 00:05:09 -07:00
Chris Lu	b465095db1	shell: add volume.check.disk to fix inconsistency for replicated volumes fix https://github.com/chrislusf/seaweedfs/issues/1923	2021-03-22 00:03:16 -07:00
Chris Lu	df461402cc	ensure entry attribute is not nill fix https://github.com/chrislusf/seaweedfs/issues/1926	2021-03-21 22:27:30 -07:00
Chris Lu	102a951377	refactor, split into 2 files	2021-03-21 13:05:53 -07:00
Chris Lu	f6e2566ee5	better logs for cookie mismatch	2021-03-20 12:02:57 -07:00
Chris Lu	27e24162f2	minor	2021-03-20 06:37:03 -07:00
Chris Lu	41cc6e661b	fs.mv ensures there are 2 arguments	2021-03-20 06:34:13 -07:00
Chris Lu	b2c1c209a5	refactor	2021-03-20 06:34:13 -07:00
Chris Lu	e9b1853d63	go fmt	2021-03-19 01:31:56 -07:00
Chris Lu	b1a86cf808	s3: copy object to itself fix https://github.com/chrislusf/seaweedfs/issues/1922	2021-03-19 01:31:49 -07:00
Konstantin Lebedev	bf94149920	add line	2021-03-17 23:41:34 +05:00
Konstantin Lebedev	c5705e7a35	dir + slash key does not exist https://github.com/chrislusf/seaweedfs/issues/1917	2021-03-17 23:40:42 +05:00
Konstantin Lebedev	eb54993a4e	Merge branch 'upstreamMaster' into check_chunkviews_mr # Conflicts: # weed/filer/filechunk_manifest.go # weed/filer/stream.go # weed/replication/repl_util/replication_util.go # weed/util/fasthttp_util.go	2021-03-16 15:29:49 +05:00
Chris Lu	9672f9e1b2	2.34	2021-03-16 03:01:15 -07:00
Chris Lu	69694a17be	reverting `7d57664c2d`	2021-03-16 02:59:26 -07:00
Chris Lu	c00dd5e62e	report error if entry not found	2021-03-16 02:55:49 -07:00
Konstantin Lebedev	06da02739d	CheckAllChunkViews() for HEAD requests only	2021-03-16 14:15:17 +05:00
Chris Lu	91a3ac9731	2.33	2021-03-16 00:36:06 -07:00
Chris Lu	4b1ed227d1	revert fasthttp changes related to https://github.com/chrislusf/seaweedfs/issues/1907	2021-03-16 00:33:14 -07:00
Chris Lu	10164d0386	Merge pull request #1905 from wuh-fnst/s3ListCounterErr s3: count correction for the number of files in the dir	2021-03-15 20:54:58 -07:00
Chris Lu	22a07a12d2	[mount] [regression] `filer.path` option results in empty mount in 2.32 fix https://github.com/chrislusf/seaweedfs/issues/1906	2021-03-15 20:46:25 -07:00
wuh-fnst	050a7f9599	s3: count correction for the number of files in the dir	2021-03-16 09:22:03 +08:00
Konstantin Lebedev	3a3699867b	Status PartialContent for Content-Range response	2021-03-15 23:30:22 +05:00
Konstantin Lebedev	90510e3137	If WriteHeader is not called explicitly, the first call to Write will trigger an implicit WriteHeader(http.StatusOK). WriteHeader: Only one header may be written. Go does not currently!	2021-03-15 21:22:59 +05:00
Konstantin Lebedev	46b9f5cff4	add debug logging	2021-03-15 19:33:33 +05:00
Konstantin Lebedev	7194a5e7bf	avoid http error: superfluous response.WriteHeader https://github.com/chrislusf/seaweedfs/issues/1838	2021-03-15 18:52:59 +05:00
Chris Lu	9f00f95bfb	2.32	2021-03-14 21:32:00 -07:00
Chris Lu	a4cfffc264	shell: fix moving volume, volume server evacuate fix https://github.com/chrislusf/seaweedfs/issues/1534	2021-03-14 21:29:55 -07:00
Chris Lu	ca100568f7	fix parsing	2021-03-14 20:50:14 -07:00
Chris Lu	352ba23f83	revert previous change revert `29e62aba00`	2021-03-14 20:49:56 -07:00
Chris Lu	29e62aba00	possible fix for volume balance address https://github.com/chrislusf/seaweedfs/issues/1534	2021-03-14 20:23:19 -07:00
Chris Lu	35d939a1d2	go fmt	2021-03-14 13:21:02 -07:00
Chris Lu	2d4c2db81d	filer: leveldb, rocksdb auto create store directory fix https://github.com/chrislusf/seaweedfs/issues/1901	2021-03-14 13:20:14 -07:00
Chris Lu	6d3a96eb56	filer: mysql2, postgres2 trigger actions on bucket creation and deletion fix https://github.com/chrislusf/seaweedfs/issues/1877	2021-03-13 22:07:39 -08:00
Chris Lu	cb423312a4	prevent nil volume message	2021-03-13 11:05:29 -08:00
Chris Lu	972327f966	prevent nil volume nm	2021-03-13 11:04:51 -08:00
Chris Lu	2a68ddb963	default to empty host to bind to tcp4 and tcp6	2021-03-12 15:32:51 -08:00
Chris Lu	5fbcaaf37a	fullpath() test is not needed	2021-03-12 09:45:25 -08:00
Chris Lu	7d57664c2d	mount: internals switch to filer.Entry instead of protobuf	2021-03-12 00:36:38 -08:00
wuh-fnst	4c1d945e46	make List correctly judge whether it is the last file	2021-03-12 11:22:26 +08:00
Chris Lu	cca66c7fbe	print out the url	2021-03-11 18:39:45 -08:00
Chris Lu	7f887c78d2	fix	2021-03-11 18:18:16 -08:00
Chris Lu	fcc52d1e4e	shell: add s3.clean.uploads to clean up stale multipart uploads fix https://github.com/chrislusf/seaweedfs/issues/1855	2021-03-11 18:05:24 -08:00
Chris Lu	40dc5ac904	mount: add a bit retry when connecting to filer during startup	2021-03-11 16:36:43 -08:00
Chris Lu	aba47fd9e8	reduce repeated calls to dir.FullPath()	2021-03-11 14:08:20 -08:00
Chris Lu	d084334ffd	Merge pull request #1883 from kmlebedev/passDelNonEmptyFolder s3 delete-objects pass OK for fail to delete non-empty folder	2021-03-11 11:01:44 -08:00
Chris Lu	0e35836cc1	Merge pull request #1890 from kmlebedev/ZeroDataSize error if read chunk zero data size	2021-03-11 10:51:48 -08:00
Konstantin Lebedev	58cdcc6d6e	error if urls not found	2021-03-11 23:34:36 +05:00
Chris Lu	79280e1513	escape special characters fix https://github.com/chrislusf/seaweedfs/issues/1884	2021-03-11 09:49:40 -08:00
Konstantin Lebedev	4db21012c1	error if read chunk zero data size	2021-03-11 22:38:59 +05:00
wuheng	828fbf3fb7	s3: "isLast" returns true when the file does not exist	2021-03-11 15:20:50 +08:00
Chris Lu	5d446673a6	fix error printing	2021-03-10 17:13:04 -08:00
Chris Lu	ca546f47e1	s3: escape object key if containing special characters fix https://github.com/chrislusf/seaweedfs/issues/1884	2021-03-10 13:19:28 -08:00
Chris Lu	105fcd7bfc	Merge pull request #1872 from kmlebedev/grpc_auth TLS allowed CommonNames	2021-03-10 11:17:49 -08:00
Konstantin Lebedev	bb3af2d70b	no directoriesWithDeletion	2021-03-10 19:52:41 +05:00
Konstantin Lebedev	15b018da34	s3 delete-objects pass OK for fail to delete non-empty folder	2021-03-10 19:41:35 +05:00
Konstantin Lebedev	348e21a08c	add comments	2021-03-10 14:42:39 +05:00
Konstantin Lebedev	831953c55c	allowed wildcard domain	2021-03-10 14:02:13 +05:00
Konstantin Lebedev	4bf93d6e63	comma-separated	2021-03-10 12:43:13 +05:00
Konstantin Lebedev	0e02f7e258	comma-separated SSL certificate common names	2021-03-10 12:42:44 +05:00
Chris Lu	737bde5ab7	refactoring, remove duplicated code	2021-03-09 23:23:01 -08:00
Chris Lu	5ba4b479f8	properly lock file.entry object fix https://github.com/chrislusf/seaweedfs/issues/1882	2021-03-09 23:08:38 -08:00
Chris Lu	5be83dd4a9	add comments	2021-03-09 14:13:48 -08:00
Chris Lu	9d8ca513d9	refactor	2021-03-09 13:21:26 -08:00
Chris Lu	f5b5e4da2d	2.31	2021-03-09 12:52:16 -08:00
Chris Lu	828f6e9f4d	volume: auto add missing vif files fix https://github.com/chrislusf/seaweedfs/issues/1878	2021-03-09 12:09:32 -08:00
Chris Lu	387c6f4218	Revert "delete the folder object in multi_object_delete" This reverts commit `2e89c8c9ae`.	2021-03-09 10:07:27 -08:00
qieqieplus	5b16820924	fix: collection stats won't update if all volumes expired at same iteration	2021-03-09 18:49:45 +08:00
Konstantin Lebedev	190fada1ef	TLS allowed commonNames	2021-03-08 21:39:44 +05:00
Konstantin Lebedev	c6d3735605	permitCommonNames https://github.com/chrislusf/seaweedfs/issues/1841 https://jbrandhorst.com/post/grpc-auth/	2021-03-08 13:16:17 +05:00
Chris Lu	d888e9a90f	2.30	2021-03-07 15:01:19 -08:00
Chris Lu	60686a0bb7	make tcp optional	2021-03-07 14:45:36 -08:00
Chris Lu	3b2a9c98ef	fix nil	2021-03-07 11:45:35 -08:00
Chris Lu	df72dc206d	stats collect chunk upload retry count	2021-03-07 11:26:15 -08:00
Chris Lu	726edab054	avoid nil when closing an index fix https://github.com/chrislusf/seaweedfs/issues/1870	2021-03-07 11:03:09 -08:00
Chris Lu	bdfed16d42	avoid nil exception fix https://github.com/chrislusf/seaweedfs/issues/1869	2021-03-07 01:49:06 -08:00
Chris Lu	1bd880dcdb	adds tcp writes benchmark	2021-03-06 14:26:27 -08:00
Chris Lu	9d402ebe9f	refactoring	2021-03-06 14:26:27 -08:00
Chris Lu	e9eddfb8a2	remove wrong writes	2021-03-06 14:26:27 -08:00
Chris Lu	8363be8548	trim out trailing return character	2021-03-06 14:26:27 -08:00
Chris Lu	38fc200e56	CRCWriter consistent with CRC	2021-03-06 14:26:27 -08:00
Chris Lu	1444e9d275	migrated multi host connection pool from godropbox package removing unneeded dependencies, which involved etcd versions.	2021-03-06 14:26:27 -08:00
Patrick Schmidt	7413d59750	Fix EC shard count logic This fixes the calculation of the amount of EC shards a node holds. Previously a global counter was increased, but also used inside the loop to apply disk usage deltas. This led to wrong absolute numbers. The fix is to apply only deltas of single EC shards per iteration.	2021-03-05 12:50:58 +01:00
Chris Lu	400de380f4	volume server: support tcp direct put/get/delete	2021-03-05 02:29:38 -08:00
Chris Lu	2e89c8c9ae	delete the folder object in multi_object_delete fix https://github.com/chrislusf/seaweedfs/issues/1857	2021-03-04 14:03:40 -08:00
Chris Lu	c5cb346a08	fix compilation error	2021-03-04 12:36:20 -08:00
Chris Lu	f5cc96f289	skip checking master peers if not starting master	2021-03-04 12:31:26 -08:00
Chris Lu	40ff30b83f	adjust help message	2021-03-03 02:12:46 -08:00
Chris Lu	c0842fe99f	add filer.meta.backup command	2021-03-03 02:02:29 -08:00
Chris Lu	0c5c51eb98	adjust help message	2021-03-02 23:07:39 -08:00
Chris Lu	bd727b7b53	go fmt	2021-03-02 20:59:56 -08:00
Chris Lu	e982b06bcd	support IPv6	2021-03-02 20:59:39 -08:00
Chris Lu	bcf32591b7	reduce possibility of nil entry	2021-03-02 13:33:56 -08:00
Chris Lu	be9c7c21ec	reduce possibility of nil file entry	2021-03-02 11:17:17 -08:00
Chris Lu	02e146f3e1	return err	2021-03-02 08:54:18 -08:00
Chris Lu	5511722420	s3: list permission is needed to list my buckets fix https://github.com/chrislusf/seaweedfs/issues/1837	2021-03-01 12:41:55 -08:00
Chris Lu	c46d16b2f3	skip network error when remote volumes start	2021-03-01 01:20:06 -08:00
Chris Lu	6e43e8ce17	add logs for remote file loading	2021-03-01 00:48:59 -08:00
Chris Lu	015d16f43f	add vif file versions in case loading superblock fails	2021-03-01 00:48:30 -08:00
Chris Lu	d680676d45	skip already loaded backends	2021-03-01 00:47:03 -08:00
Chris Lu	540441fd38	go fmt	2021-02-28 20:34:14 -08:00
Chris Lu	c9722dceb2	adjust log level	2021-02-28 20:26:55 -08:00
Chris Lu	1b1c018165	adjust text	2021-02-28 19:02:43 -08:00
Chris Lu	2cca07b44c	cloud tier to non-AWS s3 gateways	2021-02-28 18:59:09 -08:00
Chris Lu	f29ae0db76	2.29	2021-02-28 18:08:26 -08:00
bingoohuang	5cdff56731	set default env prefix to WEED_	2021-03-01 09:35:23 +08:00
Chris Lu	e52c94640e	filer.backup: added to replace filer.replicate	2021-02-28 16:22:27 -08:00
Chris Lu	014a31d11a	minor	2021-02-28 16:21:09 -08:00
Chris Lu	f2fcb77808	local file sink: create backup files with permission 0755	2021-02-28 16:20:47 -08:00
Chris Lu	984fdd6192	always use non bucket prefixing url	2021-02-28 16:20:13 -08:00
Chris Lu	9abb041763	filer source: support filerProxy mode	2021-02-28 16:19:47 -08:00
Chris Lu	678c54d705	data sink: add incremental mode	2021-02-28 16:19:03 -08:00
Chris Lu	4ff2c5c4c9	rename file	2021-02-28 16:14:21 -08:00
Chris Lu	3b76a51f5f	add help message	2021-02-27 12:15:49 -08:00
Chris Lu	70434df105	use "options"	2021-02-27 12:12:53 -08:00
Chris Lu	d3c31c69a7	avoid confusion: conf and config are too similar	2021-02-27 12:08:09 -08:00
Chris Lu	543c8daa6d	Merge pull request #1844 from kmlebedev/processRangeRequest	2021-02-26 09:18:22 -08:00
Patrick Schmidt	5f7b024891	Show the real disk usage in stats calls Currently the file size of only one volume location is taken into account in the stats. This commit multiplies the disk usages by the amount of nodes holding a replica of the volume. This will yield the expected amount of disk usage and matches the total size calculations from before.	2021-02-26 13:58:40 +01:00
Konstantin Lebedev	ee21c0042e	log error write entry	2021-02-26 14:18:01 +05:00
Chris Lu	7ba75e3d5a	filer: do not return no content for empty files fix https://github.com/chrislusf/seaweedfs/issues/1831 fix https://github.com/chrislusf/seaweedfs/issues/1830	2021-02-23 12:25:27 -08:00
Chris Lu	9edd964627	volume.tier.move: avoid repeated move for replicated volumes fix https://github.com/chrislusf/seaweedfs/issues/1792#issuecomment-784139348	2021-02-23 03:49:14 -08:00
Chris Lu	37f104f88f	2.28	2021-02-22 22:54:34 -08:00
Chris Lu	90cdf9dcac	avoid conflict with "weed scaffold -config=xxx"	2021-02-22 16:57:27 -08:00
Chris Lu	72b0d9d8c4	avoid unnecessary user home checking	2021-02-22 16:50:56 -08:00
Chris Lu	2270737344	volume: avoid fixed vacuum timeout for large volumes 1GB for 3 minutes, about 5.7MB/s	2021-02-22 12:52:37 -08:00
Chris Lu	44bdfb2d15	filer: avoid encryption and compression at the same time fix https://github.com/chrislusf/seaweedfs/issues/1828	2021-02-22 12:22:49 -08:00
Chris Lu	62191b08ea	disk type support custom tags	2021-02-22 02:03:12 -08:00
Chris Lu	30b30b8fe0	volume.tier.move: passing non-empty disk type	2021-02-22 01:59:03 -08:00
Chris Lu	5da63e045e	avoid moving to another server with the same volume id	2021-02-22 01:44:18 -08:00
Chris Lu	6a4546d2c0	shell: add volume.tier.move	2021-02-22 01:30:07 -08:00
Chris Lu	1c233ad986	refactoring	2021-02-22 00:28:42 -08:00
Chris Lu	151c281f36	2.27	2021-02-21 19:29:27 -08:00
Chris Lu	258e93bc86	Merge pull request #1823 from bingoohuang/master feature: support command line arguments in a configuration file	2021-02-20 22:35:18 -08:00
Chris Lu	03c643aa6b	fix test fileSize need to be divided by 8	2021-02-20 21:19:21 -08:00
Chris Lu	7635f6b9fa	disk file avoid file.Stat()	2021-02-20 20:06:06 -08:00
bingoo	1af6c96cd0	fix log help	2021-02-21 10:46:01 +08:00
Chris Lu	a2383b3b12	Revert "Revert "avoid file.Stat()"" This reverts commit `7ef4c24f18`.	2021-02-20 12:42:09 -08:00
Chris Lu	7ef4c24f18	Revert "avoid file.Stat()" This reverts commit `98c93ca465`.	2021-02-20 12:39:33 -08:00
Chris Lu	1ad3200094	skip seek() when index file writes	2021-02-20 12:39:25 -08:00
Chris Lu	98c93ca465	avoid file.Stat() avoid one Syscall, but did not help on performance though	2021-02-20 09:44:17 -08:00
bingoohuang	ee082ae402	revert comment temporarily	2021-02-20 16:59:38 +08:00
bingoohuang	ee7cdf3668	revert ErrXyz to ErrorXyz temporarily.	2021-02-20 16:58:08 +08:00
bingoohuang	50df484d86	support command line arguments in a configuration file so we can `weed master -config=master.conf`, `weed volume -config=volume.conf` and etc.	2021-02-20 16:52:57 +08:00
bingoohuang	f69356f589	Merge pull request #3 from chrislusf/master	2021-02-20 16:45:02 +08:00
Chris Lu	565f7a6e72	Update data_node.go	2021-02-19 14:22:36 -08:00
Chris Lu	a37473ae60	add back volume ids address https://github.com/chrislusf/seaweedfs/issues/1792#issuecomment-782339576	2021-02-19 14:22:12 -08:00
Chris Lu	a1210d1e8d	Revert "volume: avoid file.stat(), file.seek() if possible during writes" This reverts commit `c78409a598`.	2021-02-19 03:56:27 -08:00
Chris Lu	b961cd6208	add WIP message	2021-02-19 03:39:19 -08:00
Chris Lu	f465d63b5d	grpc should fail when heart beating to master fix https://github.com/chrislusf/seaweedfs/issues/1820	2021-02-19 03:37:29 -08:00
Chris Lu	c576ad04ac	fix volume server display for volumes	2021-02-19 01:38:56 -08:00
bingoohuang	7ffe736d20	renaming to ErrXyz	2021-02-19 15:59:55 +08:00
Chris Lu	c78409a598	volume: avoid file.stat(), file.seek() if possible during writes	2021-02-18 23:22:54 -08:00
Chris Lu	29575dadc5	remove extra files	2021-02-18 19:14:58 -08:00
Chris Lu	73958e357d	add descriptive error if no free volumes	2021-02-18 19:10:20 -08:00
Chris Lu	776f497469	filer: fs.configure should try to read from entry.content also related to https://github.com/chrislusf/seaweedfs/issues/1792	2021-02-18 17:07:02 -08:00
Chris Lu	e1992c83de	webdav adds disktype	2021-02-18 15:45:44 -08:00
Chris Lu	c2ad6f1047	webdav add replication setting fix https://github.com/chrislusf/seaweedfs/issues/1817	2021-02-18 12:15:09 -08:00
Chris Lu	816ef0ed30	add more help message fix https://github.com/chrislusf/seaweedfs/issues/1816	2021-02-18 10:34:49 -08:00
bingoohuang	352ac2f271	Merge remote-tracking branch 'origin/master'	2021-02-18 14:05:51 +08:00
bingoohuang	eab6e31d34	use backticks instead of double quotes to avoid escaped additionally in regex	2021-02-18 14:05:28 +08:00
Chris Lu	3575d41009	go fmt	2021-02-17 20:57:08 -08:00
Chris Lu	6daa932f5c	refactoring to get master function, instead of passing master values directly this will enable retrying later	2021-02-17 20:55:55 -08:00
Chris Lu	dd9f3a0104	add sleep between upload retries	2021-02-17 20:55:11 -08:00
Chris Lu	aa17311063	s3: set filename when uploading file chunks	2021-02-17 20:54:53 -08:00
Chris Lu	3f8b0da677	filer: do not print password on error fix https://github.com/chrislusf/seaweedfs/issues/1809	2021-02-17 02:13:52 -08:00
Chris Lu	cd866664a8	skip JWT if fileId is empty related to https://github.com/chrislusf/seaweedfs/issues/1808	2021-02-16 15:39:12 -08:00
Chris Lu	632c94d438	Merge branch 'extend_to_disk_type'	2021-02-16 11:06:13 -08:00
Chris Lu	0611233f16	adjust printing	2021-02-16 10:55:30 -08:00
Chris Lu	68775d29e3	fix tests	2021-02-16 10:51:03 -08:00
Chris Lu	7403cd43c5	only use "" for hdd to avoid two values for the same thing	2021-02-16 10:50:42 -08:00
Chris Lu	b314d78e97	fix print	2021-02-16 10:48:28 -08:00
Chris Lu	53ca7e66ef	avoid dead lock	2021-02-16 10:48:16 -08:00
Chris Lu	38efc6f572	simplify	2021-02-16 10:47:32 -08:00
Chris Lu	ad70107eb4	ec balance among nodes with HDD	2021-02-16 06:22:50 -08:00
Chris Lu	3097b9a9b7	fix existence checking	2021-02-16 05:59:43 -08:00
Chris Lu	ebf320ec22	avoid divided by zero	2021-02-16 05:59:24 -08:00
Chris Lu	a0c6db361c	avoid nil	2021-02-16 05:33:38 -08:00
Chris Lu	36f95e50a9	avoid possible nil disk info	2021-02-16 05:13:48 -08:00
Chris Lu	43101ccea0	move to the empty nodes first	2021-02-16 04:27:16 -08:00
Chris Lu	38bbef7ec1	avoid nil	2021-02-16 04:16:46 -08:00
Chris Lu	cb9cc29518	volume.list display; fix updating maxVolumeCount for disk	2021-02-16 03:55:24 -08:00
Chris Lu	3fe628f04e	use hdd instead of empty string	2021-02-16 03:03:00 -08:00
Chris Lu	f8446b42ab	this can compile now!!!	2021-02-16 02:47:02 -08:00
Chris Lu	b9b5b932c5	filer: postgres2 mysql2 avoid repeatedly creating tables	2021-02-15 16:19:24 -08:00
Chris Lu	71f0c19515	2.26	2021-02-15 13:38:38 -08:00
Chris Lu	8b169aa47c	s3: Critical - S3 Delete Multiple Objects, delete the bucket fix https://github.com/chrislusf/seaweedfs/issues/1806	2021-02-15 13:38:29 -08:00
Chris Lu	50202d1391	Merge pull request #1805 from LazyDBA247-Anyvision/master better postgresql connection pool management	2021-02-14 22:26:29 -08:00
Chris Lu	8726ef34c1	2.25	2021-02-14 22:23:43 -08:00
LazyDBA247-Anyvision	7f458d5e78	better postgres connection pool management adding SetConnMaxLifetime configuration (https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime) to enable refresh of stale connections.	2021-02-15 07:45:09 +02:00
Chris Lu	0bc3a1f9e8	disk type only supports hdd and ssd, not ready for random tags yet	2021-02-14 11:38:43 -08:00
LazyDBA247-Anyvision	51b4963e2e	postgres2 & memsql2 add escape (quote identifiers) for the dynamic sql so tables (collections) with special characters will work.	2021-02-14 13:14:36 +02:00
Chris Lu	876dbe3d26	collect disk types	2021-02-13 23:25:16 -08:00
Chris Lu	861f8fab9b	refactoring	2021-02-13 22:34:12 -08:00
Chris Lu	881f05f5a0	adjust help message	2021-02-13 16:13:52 -08:00
Chris Lu	ef76365ec2	adjust help message	2021-02-13 15:47:08 -08:00
Chris Lu	7ce647f27e	support customizable disk type	2021-02-13 15:42:42 -08:00
Chris Lu	4bd8a692d8	disk type can be generic tags	2021-02-13 13:50:14 -08:00
Chris Lu	712b3e9e53	mount: release need to avoid nil reader fix https://github.com/chrislusf/seaweedfs/issues/1803	2021-02-13 13:39:37 -08:00
Chris Lu	4ce56bac08	fix format error	2021-02-12 06:44:26 -08:00
Chris Lu	0604afcda7	mount: remove unnecessary flush on release	2021-02-12 06:01:04 -08:00
Chris Lu	7d9dc3c6a2	use fasthttp lib to read	2021-02-12 05:59:52 -08:00
Chris Lu	487e435679	adjust http max idle connections per host related to https://github.com/chrislusf/seaweedfs/issues/1802	2021-02-12 03:47:15 -08:00
Chris Lu	0f426ce34d	Merge pull request #1801 from kmlebedev/recoveringRabbitMQ Do reconnect to RabbitMQ	2021-02-11 01:05:03 -08:00
Konstantin Lebedev	94eac4f00e	Do reconnect to RabbitMQ https://github.com/google/go-cloud/issues/2958 https://github.com/chrislusf/seaweedfs/issues/1773	2021-02-11 13:59:36 +05:00
Chris Lu	885ca34748	volume: fail fast if idx files are missing fix https://github.com/chrislusf/seaweedfs/issues/1796	2021-02-11 00:44:40 -08:00
Chris Lu	a0e84c4fbc	go fmt	2021-02-10 23:41:05 -08:00
Chris Lu	c959aaa19b	mount: load directory based on correct filer path fix https://github.com/chrislusf/seaweedfs/issues/1799	2021-02-10 23:14:18 -08:00
Konstantin Lebedev	0cfed8c3cb	restart replication if RabbitMQ connection closed https://github.com/google/go-cloud/issues/2958	2021-02-10 15:29:05 +05:00
Chris Lu	770393a48c	volume: add capability to change disk type when moving a volume	2021-02-09 23:58:08 -08:00
Chris Lu	821c46edf1	Merge branch 'master' into support_ssd_volume	2021-02-09 11:37:07 -08:00
Chris Lu	a833021132	fix refactoring left over	2021-02-07 23:03:03 -08:00
Chris Lu	5e4b5109dd	2.24	2021-02-07 22:29:43 -08:00
Chris Lu	15c60cbb26	close the grpc connection after 10 hours related to https://github.com/chrislusf/seaweedfs/issues/1782	2021-02-07 03:50:01 -08:00
bingoohuang	7256902fb0	fix typo offset.ToAcutalOffset to offset.ToActualOffset	2021-02-07 12:11:51 +08:00
bingoohuang	94ea3bd3a5	renaming NeedleMapType to NeedleMapKind	2021-02-07 09:00:03 +08:00
Chris Lu	c3af72d950	fier store: fix elastic search regression fix https://github.com/chrislusf/seaweedfs/issues/1774	2021-02-04 01:30:14 -08:00
Konstantin Lebedev	2fb4c60540	fix log warn	2021-02-03 21:34:32 +05:00
Konstantin Lebedev	24ec17219b	RabbitMQ delay retry with Dead Letter Exchange https://github.com/chrislusf/seaweedfs/issues/1773 https://github.com/google/go-cloud/issues/2952	2021-02-03 21:32:30 +05:00
Chris Lu	a331bbb3ae	filer: should return 204 on DELETE to nonexistent file related to https://github.com/chrislusf/seaweedfs/issues/1776 https://github.com/chrislusf/seaweedfs/issues/1160	2021-02-03 00:40:31 -08:00
Chris Lu	5353e38469	S3: should return 204 on DELETE to nonexistent file fix https://github.com/chrislusf/seaweedfs/issues/1776	2021-02-03 00:35:44 -08:00
Chris Lu	2396ac234c	filer: fix elastic search pagination possible fix for https://github.com/chrislusf/seaweedfs/issues/1774	2021-02-02 11:34:12 -08:00
Chris Lu	609daaf387	s3: DeleteMultipleObjectsHandler clean up leftover empty folders fix https://github.com/chrislusf/seaweedfs/issues/1772	2021-02-01 10:49:17 -08:00
Chris Lu	d67ccb66c0	webdav: can start together with "weed server" or "weed filer"	2021-01-31 22:16:52 -08:00
Chris Lu	76430790b9	webdav: cache to version specific folder	2021-01-31 22:02:03 -08:00
Chris Lu	318a3d2efc	2.23	2021-01-31 20:18:06 -08:00
Chris Lu	4be51c0701	filer: leveldb and hbase may miss files when listing large directories more than 1024 fix https://github.com/chrislusf/seaweedfs/issues/1768	2021-01-31 20:11:44 -08:00
Chris Lu	1102ae32c4	fix concurrent map reads	2021-01-31 18:26:26 -08:00
Chris Lu	314dc1c957	filer: etcd store fix listing fix https://github.com/chrislusf/seaweedfs/issues/1767	2021-01-30 13:38:44 -08:00
Chris Lu	d475c89fcc	go fmt	2021-01-28 15:23:46 -08:00
Chris Lu	19295600f9	mount: change option name to volumeServerAccess, with publicUrl and filerProxy modes	2021-01-28 15:23:16 -08:00
Chris Lu	990fa69bfe	add back AdjustedUrl() related code	2021-01-28 14:36:29 -08:00
Chris Lu	9292796ec2	fmt	2021-01-28 14:29:14 -08:00
Chris Lu	c2bf1a88ac	delete from the deepest directory first when checking empty folders	2021-01-28 14:28:40 -08:00
Chris Lu	e9d8201925	s3: batch purge empty folders	2021-01-28 13:20:06 -08:00
Chris Lu	cf252fc0cd	mount: report error when Flush() The error in Release() is not working. See https://github.com/jaderhs/libfuse/blob/master/FAQ related to https://github.com/chrislusf/seaweedfs/issues/1765	2021-01-28 04:46:37 -08:00
Chris Lu	9a06c35da4	replicate: incremental sink only contains new and updated files address `da08402ba2`	2021-01-28 02:39:22 -08:00
Chris Lu	da08402ba2	replicate: use creation time for local incremental file sink related to https://github.com/chrislusf/seaweedfs/pull/1762	2021-01-28 02:17:41 -08:00
Chris Lu	822f1ade9d	Merge pull request #1762 from kmlebedev/backupsink replication to create time date directory	2021-01-28 02:03:33 -08:00
Konstantin Lebedev	be1062b7fc	rename in scaffold	2021-01-28 14:59:20 +05:00
Konstantin Lebedev	02fdc0a333	rename backup to local_incremental and use mtime	2021-01-28 14:56:13 +05:00
Chris Lu	5f72c388bc	Merge pull request #1764 from danlsgiga/handle-umask Use provided umask on mount	2021-01-28 01:09:20 -08:00
Daniel Santos	d38613c49a	match upstream	2021-01-27 20:56:41 -07:00
Chris Lu	b81956bcb5	mount: invalidate kernel cache when mounted to a filer path fix https://github.com/chrislusf/seaweedfs/issues/1752#issuecomment-768178422	2021-01-27 10:28:37 -08:00
Konstantin Lebedev	6b54ff9912	replication to create time date directory	2021-01-27 15:01:33 +05:00
Chris Lu	f20ec82a28	update help message see https://github.com/chrislusf/seaweedfs/issues/1758#issuecomment-768123224	2021-01-27 00:48:31 -08:00
Chris Lu	1045ab8b07	avoid error message	2021-01-27 00:21:59 -08:00
Chris Lu	0488c84bea	adjust help message	2021-01-27 00:17:52 -08:00
Konstantin Lebedev	3634811408	replication to todays date directory	2021-01-27 12:45:58 +05:00
Chris Lu	5e07afb0f0	Merge pull request #1759 from kmlebedev/sink.local replication to local disk storage	2021-01-26 22:31:54 -08:00
Chris Lu	b3f66199db	shell: volume.fix.replication adds collectionPattern with wildcard characters fix https://github.com/chrislusf/seaweedfs/issues/1758	2021-01-26 22:30:39 -08:00
Daniel Santos	0c64d60fdb	Fix log message with correct mode	2021-01-26 21:45:52 -07:00
Daniel Santos	0aaaa9b470	Fix var assignment	2021-01-26 21:06:08 -07:00
Daniel Santos	5a761bb7a4	Enforce umask correctly	2021-01-26 20:42:10 -07:00
Chris Lu	ad2a20c8a5	notification add ack and nack	2021-01-26 11:08:44 -08:00
Konstantin Lebedev	612b7975a1	replication to local disk storage	2021-01-26 22:50:25 +05:00
Chris Lu	3a1d3d3413	mount: properly invalidate kernel node cache entry fix https://github.com/chrislusf/seaweedfs/issues/1752	2021-01-26 02:50:53 -08:00
Chris Lu	cc1f3907ff	filer.sync: default filerProxy to true	2021-01-25 10:14:42 -08:00
Chris Lu	62ebb917e0	mount: avoid the need to specify different cache directory for different mount fix https://github.com/seaweedfs/seaweedfs-csi-driver/issues/11	2021-01-25 09:10:20 -08:00
Chris Lu	17f7c1c43f	Merge branch 'mount_from_outside_cluster'	2021-01-24 19:03:02 -08:00
Chris Lu	00707ec00f	mount: outsideContainerClusterMode proxy through filer Running mount outside of the cluster would not need to expose all the volume servers to outside of the cluster. The chunk read and write will go through the filer.	2021-01-24 19:01:58 -08:00
Chris Lu	9bcb28a3ea	avoid possible nil reader fix https://github.com/chrislusf/seaweedfs/issues/1754	2021-01-24 17:19:35 -08:00
Chris Lu	2c5eac5705	2.22	2021-01-24 14:06:09 -08:00
Chris Lu	6ca10725b8	Revert "mount: when outside cluster network, use filer as proxy to access volume servers" This reverts commit `096e088d7b`.	2021-01-24 03:15:19 -08:00
Chris Lu	096e088d7b	mount: when outside cluster network, use filer as proxy to access volume servers	2021-01-24 01:41:38 -08:00
Chris Lu	ccbdb38c89	s3: fix for listing objects if more than 1000 in the folder many existing tools have max key set to 1000	2021-01-24 00:27:52 -08:00
Chris Lu	80b8692688	filer.sync: replicate outside of either cluster, only need to see filers	2021-01-24 00:01:44 -08:00
Chris Lu	f8dbb03bdd	filer: append operation returns final file size	2021-01-23 04:19:49 -08:00
Chris Lu	759482e2ac	filer: proxy all http methods to volume servers	2021-01-23 03:52:38 -08:00
Chris Lu	937cfacc01	filer: add "proxyToFileId" to reverse proxy to a volume server	2021-01-23 03:43:48 -08:00
Chris Lu	20ef3bb8d4	mount: use direct_io to avoid OS page cache fix https://github.com/chrislusf/seaweedfs/issues/1752	2021-01-22 22:39:46 -08:00
Chris Lu	711c3f3939	filer: fix http status 206 setting fix https://github.com/chrislusf/seaweedfs/issues/1753	2021-01-22 00:26:30 -08:00
Chris Lu	024c186a48	filer.meta.tail: output json format for easier parsing	2021-01-21 22:39:25 -08:00
Chris Lu	84f05787f8	mount: limit background requests and congestion threshold	2021-01-20 23:30:14 -08:00
Chris Lu	f98817cfe6	filer: support appending to a file	2021-01-20 13:40:32 -08:00
Chris Lu	1b8e3da295	refactoring	2021-01-20 12:59:54 -08:00
Chris Lu	1a3f859c58	filer store: postgres configuration adjust text	2021-01-19 18:12:07 -08:00
Chris Lu	b434f7e4e0	filer store: postgres adjust default config, add optioanl schema	2021-01-19 18:10:36 -08:00
Chris Lu	d5add83e85	filer store: add postgres2	2021-01-19 18:07:29 -08:00
Chris Lu	52a8f1470e	filer store: add mysql2	2021-01-19 17:21:50 -08:00
Chris Lu	93b3adba98	fix bucket creation	2021-01-19 15:55:51 -08:00
Chris Lu	fa0c8d5283	fix error	2021-01-19 14:05:48 -08:00
Chris Lu	4c5b752b04	restructuring sql stores	2021-01-19 13:53:16 -08:00
Chris Lu	96354208c5	scaffold: add YugabyteDB	2021-01-19 12:35:14 -08:00
Chris Lu	ca8f793978	refactor: prepare for bucket specific tables	2021-01-19 12:34:58 -08:00
Chris Lu	90ce1eec19	add more input params	2021-01-19 11:32:55 -08:00
Chris Lu	f13c082583	add memsql to comments	2021-01-19 11:31:55 -08:00
Chris Lu	e439b65e38	fix test	2021-01-18 01:48:00 -08:00
Chris Lu	c7197470fd	2.21	2021-01-18 01:25:04 -08:00
Chris Lu	5a3386e39b	Revert "Revert "mount: fake support for socket/block/character/fifo devices"" This reverts commit `3d5cb7eb86`.	2021-01-18 01:15:07 -08:00
Chris Lu	67faa56920	Revert "Revert "mount: add more entry type"" This reverts commit `4085b79d38`.	2021-01-18 01:14:58 -08:00
Chris Lu	7119e7b491	mount: ensure reading from dirty pages	2021-01-18 01:14:42 -08:00
Chris Lu	de876c795d	minor fix	2021-01-18 01:14:27 -08:00
Chris Lu	4085b79d38	Revert "mount: add more entry type" This reverts commit `d7d907be27`.	2021-01-18 00:19:19 -08:00
Chris Lu	3d5cb7eb86	Revert "mount: fake support for socket/block/character/fifo devices" This reverts commit `61ef2d8658`.	2021-01-18 00:18:57 -08:00
Chris Lu	61ef2d8658	mount: fake support for socket/block/character/fifo devices to pass pjdfstest. A distributed file system can not really support these things anyway.	2021-01-17 23:51:10 -08:00
Chris Lu	d7d907be27	mount: add more entry type	2021-01-17 23:46:39 -08:00
Chris Lu	389426bbb7	s3: listing may repeat on the edge	2021-01-17 22:59:31 -08:00
Chris Lu	09f49d1c04	refactoring	2021-01-16 19:52:15 -08:00
Chris Lu	a4063a5437	add stream list directory entries	2021-01-15 23:56:24 -08:00
Chris Lu	01dc8a43ba	fix rocksdb	2021-01-15 18:31:29 -08:00
Chris Lu	16ad74f477	go fmt	2021-01-14 23:11:27 -08:00
Chris Lu	f002e668de	change limit to int64 in case of overflow	2021-01-14 23:10:37 -08:00
Chris Lu	19e52fd414	pass along hasMore	2021-01-14 22:49:29 -08:00
Chris Lu	c0bcf6f6e1	implement sql changes	2021-01-14 22:44:22 -08:00
Chris Lu	5d4568b91f	implement elastic changes	2021-01-14 22:42:25 -08:00
Chris Lu	893cbc8482	implement c* changes	2021-01-14 22:38:34 -08:00
Chris Lu	5ef43b9b09	implement etcd changes	2021-01-14 22:35:56 -08:00
Chris Lu	54527f0326	implement hbase changes	2021-01-14 22:34:35 -08:00
Chris Lu	b5ceffe188	implement leveldb changes	2021-01-14 22:33:05 -08:00
Chris Lu	c64bfb0e2e	implement mongodb changes	2021-01-14 22:28:51 -08:00
Chris Lu	34a846009d	implement rocksdb changes	2021-01-14 22:24:26 -08:00
Chris Lu	d2ece29fff	implement redis changes	2021-01-14 22:21:31 -08:00
Chris Lu	9a50dbcda0	chagned api	2021-01-14 22:18:38 -08:00
Chris Lu	e1c7bc66f9	change interface	2021-01-14 22:17:49 -08:00
Chris Lu	4aaa97c789	skip not found error in case of race condition	2021-01-14 19:56:14 -08:00
Chris Lu	d0826c2f62	mount: avoid showing "topics" folder fix https://github.com/chrislusf/seaweedfs/issues/1325	2021-01-14 19:03:15 -08:00
Chris Lu	698f58f7c4	filer, s3: add http status 206 as late as possible fix https://github.com/chrislusf/seaweedfs/issues/1746	2021-01-14 02:59:40 -08:00
Chris Lu	fd1d8a2a09	fix compilation	2021-01-14 00:06:15 -08:00
Chris Lu	ae812c07d6	Merge pull request #1743 from LazyDBA247-Anyvision/master MYSQL - Better Connection Pool Control	2021-01-14 00:04:44 -08:00
Chris Lu	f20743bb38	remove WIP status	2021-01-13 22:21:34 -08:00
LazyDBA247-Anyvision	8eed763b97	better mysql connection pool management adding SetConnMaxLifetime configuration (https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime) to enable refresh of connections.	2021-01-14 08:14:21 +02:00
Chris Lu	f17aa1d06c	skip directory loop if dropping the bucket	2021-01-13 13:49:04 -08:00
Chris Lu	ca73013453	leveldb3: ensure repeatable adding removing buckets	2021-01-13 13:48:48 -08:00
Chris Lu	e2c7e3fe6d	better locks	2021-01-13 13:20:33 -08:00
Chris Lu	819a85f59a	avoid viper concurrent access	2021-01-13 13:14:52 -08:00
Chris Lu	3fb2ed9093	filer.meta.tail: optionally submit metadata changes to ElasticSearch	2021-01-13 03:31:37 -08:00
Chris Lu	c5df2577f5	rename	2021-01-13 00:31:19 -08:00
Chris Lu	90c5077610	rename "weed watch" to "weed filer.meta.tail"	2021-01-12 18:48:01 -08:00
Chris Lu	5cd9c9e642	remove println	2021-01-12 18:47:38 -08:00
Chris Lu	def6d8e565	fix test	2021-01-12 12:32:54 -08:00
Chris Lu	1c7faf1100	Revert "close opened file" This reverts commit `2880160113`.	2021-01-12 11:07:14 -08:00
Chris Lu	2880160113	close opened file	2021-01-12 11:03:08 -08:00
Chris Lu	0ef43a23a7	go fmt	2021-01-12 02:30:12 -08:00
Chris Lu	629c9962e7	filer: add leveldb3 support dedicated leveldb instance for each bucket	2021-01-12 02:29:44 -08:00
Chris Lu	361043e6c1	filer store: leveldb2 fix nil entry error if not found	2021-01-12 02:28:57 -08:00
Chris Lu	cfb9342a15	avoid concurrent map updates to viper	2021-01-12 02:28:13 -08:00
Chris Lu	38d516251e	Revert "mount: avoid fatal error" This reverts commit `a4f0bac9b7`.	2021-01-11 13:50:15 -08:00
Chris Lu	a4f0bac9b7	mount: avoid fatal error	2021-01-11 13:26:59 -08:00
Chris Lu	8a7302ce37	s3: KeyCount is zero fix https://github.com/chrislusf/seaweedfs/issues/1741	2021-01-11 03:50:45 -08:00
Chris Lu	2c7148cd55	redis cluster: do not enable route by latency by default	2021-01-11 02:43:53 -08:00
Chris Lu	acc0d5cd2c	Revert "Revert "filer store: redis module upgrade"" This reverts commit `dd0171e35c`.	2021-01-11 02:30:19 -08:00
Chris Lu	dd0171e35c	Revert "filer store: redis module upgrade" This reverts commit `40161c9cec`.	2021-01-11 02:09:18 -08:00
Chris Lu	394513f598	filer: ensure seamless meta data updates	2021-01-11 02:08:55 -08:00
Chris Lu	6e12a3a490	refactoring	2021-01-11 02:08:26 -08:00
Chris Lu	62ee484d12	refactoring	2021-01-11 00:03:13 -08:00
Chris Lu	1efb51ba84	filer: change to saveToFilerLimit from cacheToFilerLimit short circuit saving small files to volume server	2021-01-10 23:14:46 -08:00
Chris Lu	f0d3b3bf93	2.20	2021-01-09 22:55:22 -08:00
Chris Lu	d3d3f2fb9b	mount: default to 128 concurrent writers	2021-01-09 22:53:37 -08:00
Chris Lu	90b117acf1	update ccache version	2021-01-08 02:17:43 -08:00
Chris Lu	e2e0e30a51	filer: redis default to false for useReadOnly and routeByLatency	2021-01-08 01:12:44 -08:00
Chris Lu	40161c9cec	filer store: redis module upgrade	2021-01-08 01:08:44 -08:00
Chris Lu	30efaa66c6	server: add option to disable master	2021-01-07 02:35:47 -08:00
Chris Lu	07f4703bfc	Merge pull request #1738 from qieqieplus/rocksdb fix rocksdb crash when list directory	2021-01-07 02:26:41 -08:00
qieqieplus	0f18592315	alter style	2021-01-07 18:21:48 +08:00
qieqieplus	c7e7431fd3	fix seek lastkey may reach EOF	2021-01-07 17:07:56 +08:00
Chris Lu	e327385644	go fmt	2021-01-06 04:22:00 -08:00
Chris Lu	2b76854641	add "weed filer.cat" to read files directly from volume servers	2021-01-06 04:22:00 -08:00
Chris Lu	95ecf0c72f	Merge pull request #1735 from qieqieplus/rocksdb ignore decode error for non-entry data	2021-01-06 02:22:18 -08:00
qieqieplus	0764fccde7	ignore decode error for non-entry data	2021-01-06 17:42:43 +08:00
Chris Lu	ab57767d4b	Merge pull request #1730 from qieqieplus/rocksdb impl: TTL per entry for rocksdb; fix package name	2021-01-05 13:24:45 -08:00
qieqieplus	b2e50f602f	fix time comparison	2021-01-05 19:36:40 +08:00
qieqieplus	bbae13e415	impl: TTL per entry for rocksdb; fix package name	2021-01-05 17:58:40 +08:00
henry	b42c5b71c1	Seaweed custom header are not visible to Vue or javascript	2021-01-05 13:01:29 +08:00
henry	1309dfe612	Seaweed custom header are not visible to Vue or javascript	2021-01-05 12:58:46 +08:00
henry	942a30f04c	remove sort	2021-01-05 12:24:32 +08:00
henry	a3c432dadc	Merge branch 'master' of https://github.com/fuyouyshengwu/seaweedfs	2021-01-05 11:17:29 +08:00
qieqieplus	ebb223c190	fix for rocksdb kv	2021-01-04 21:21:46 +08:00
chenqieqie	40b3207f18	fix #1726	2021-01-04 18:48:55 +08:00
henry	97a94eddab	Merge branch 'master' of https://github.com/fuyouyshengwu/seaweedfs	2021-01-04 14:50:01 +08:00
Chris Lu	2ce86f308e	2.19	2021-01-03 17:24:02 -08:00
Chris Lu	9c9ba3c209	nil related related to https://github.com/chrislusf/seaweedfs/issues/1676	2021-01-03 12:25:58 -08:00
Chris Lu	4b5adc60b4	upload result changes to the right output variable	2021-01-03 01:44:22 -08:00
Chris Lu	039ad101b1	manifest file also need to detect compression fix https://github.com/chrislusf/seaweedfs/issues/1724#issuecomment-753585046	2021-01-03 01:42:21 -08:00
Chris Lu	5f538c5f9d	Merge branch 'add_rocksdb'	2021-01-03 00:48:56 -08:00
Chris Lu	bdd166c031	adjust readme	2021-01-03 00:47:58 -08:00
Chris Lu	a04d8927a8	add scaffold	2021-01-03 00:44:52 -08:00
Chris Lu	4250a3a675	upload: add read default collection from master fix https://github.com/chrislusf/seaweedfs/issues/1724#issuecomment-753576071	2021-01-02 23:43:01 -08:00
Chris Lu	f4d87f8b9d	works performance seems about the same as leveldb for insert entries	2021-01-02 23:32:58 -08:00
Chris Lu	2c3c2c27d7	separate prefix from namePattern fix https://github.com/chrislusf/seaweedfs/issues/1722	2021-01-01 20:23:23 -08:00
Peter Cai	cd2a9865ed	filer: respect Content-Type on PUT This is needed for the S3 gateway to conform to the S3 protocol -- the server should always respect the Content-Type set by client.	2020-12-31 11:32:45 +08:00
Chris Lu	3433accb1b	weed upload: add TTL	2020-12-30 15:46:07 -08:00
henry	14ddd15508	1 Distinguish between folders and common files, folders are listed first 2 Add some custom headers that need to be exposed	2020-12-31 00:03:22 +08:00
Chris Lu	adef4ddc87	mount: load security keys fix https://github.com/chrislusf/seaweedfs/issues/1714	2020-12-29 11:51:38 -08:00
Chris Lu	89977123b6	shell: add shell.toml support fix https://github.com/chrislusf/seaweedfs/issues/1664	2020-12-28 15:07:18 -08:00
Chris Lu	d2ead72918	2.18	2020-12-28 12:37:10 -08:00
Chris Lu	2e6cf541e7	filer: avoid deletion error when item is not found	2020-12-28 00:18:37 -08:00
Chris Lu	6d2c979999	s3: fix regression fix https://github.com/chrislusf/seaweedfs/issues/1707	2020-12-27 21:09:45 -08:00
Chris Lu	90df52d001	s3: add command option "allowEmptyFolder" to filer and server	2020-12-27 21:09:06 -08:00
Chris Lu	da7e5aaa65	filer: optimize for less number of directory lookup bottom up directory lookup	2020-12-27 20:46:17 -08:00
henry	7458ff9523	Solve the Chinese name disorder of download file	2020-12-28 11:28:33 +08:00
Chris Lu	561a2ff0bc	s3: add option for "alllowEmptyFolder"	2020-12-27 10:50:27 -08:00
Chris Lu	aa020ee7e7	s3: restore V2 auth type The added headers were included in calculating the signatures, failing the comparison	2020-12-26 22:01:16 -08:00
Chris Lu	da87f6b265	remove unused code	2020-12-26 15:21:12 -08:00
Chris Lu	0a7c5f85a9	filer: add namePattern to search in current folder	2020-12-26 15:05:31 -08:00
Chris Lu	515b7632d7	2.17	2020-12-25 23:58:22 -08:00
Chris Lu	0a067944cc	filer: add retries during volume moving fix https://github.com/chrislusf/seaweedfs/issues/1704	2020-12-25 02:32:55 -08:00
Chris Lu	090f85be4b	s3: support config action Admin:bucket	2020-12-25 00:38:56 -08:00
Chris Lu	75613b2cc7	Merge branch 'add_hbase'	2020-12-24 12:13:05 -08:00
Chris Lu	0e016bc7bd	hbase add ttl	2020-12-24 12:10:35 -08:00
Chris Lu	8e48a235e2	s3: avoid looping if the directory is empty fix https://github.com/chrislusf/seaweedfs/issues/1701	2020-12-24 11:34:52 -08:00
Chris Lu	1620de08ae	added more logs	2020-12-24 01:52:06 -08:00
Chris Lu	a09cd28986	add debug messages	2020-12-24 01:42:15 -08:00
Chris Lu	75c6edba9e	filer: hbase add enabled flag	2020-12-24 00:19:16 -08:00
Chris Lu	9bf6c10505	fix prefix	2020-12-24 00:18:59 -08:00
Chris Lu	2fea8cfa0f	fix compilation	2020-12-23 23:54:15 -08:00
Chris Lu	53bc1ea25b	fix compilation	2020-12-23 23:53:46 -08:00
Chris Lu	b5e2be635a	adjust for directory listing	2020-12-23 23:49:22 -08:00
Chris Lu	c4a202ec41	fix wrong column family	2020-12-23 23:23:05 -08:00
Chris Lu	c3d1b3b5aa	hook up	2020-12-23 21:49:01 -08:00
Chris Lu	94e3757c08	fix	2020-12-23 21:45:16 -08:00
Chris Lu	64c48c9724	seems compiling	2020-12-23 21:44:53 -08:00
Chris Lu	15da5834e1	Merge branch 'master' into support_ssd_volume	2020-12-23 14:37:37 -08:00
Chris Lu	3be3635799	just log error when checking empty dir failed	2020-12-23 14:36:48 -08:00
Chris Lu	0ca9d89589	s3: break loop if error fix #1701	2020-12-23 14:34:59 -08:00
Chris Lu	132f275d04	s3: log errors instead of stopping when checking isDirectoryAllEmpty has error	2020-12-23 12:21:21 -08:00
Chris Lu	db62090f88	s3: break loop if error fix https://github.com/chrislusf/seaweedfs/issues/1701	2020-12-23 12:14:51 -08:00
Baptiste Mille-Mathias	385a4dec30	[shell] Clarify bucket replication setting order	2020-12-23 09:35:24 +01:00
Chris Lu	1c7e1295dc	Merge branch 'master' into support_ssd_volume	2020-12-22 17:44:52 -08:00
Chris Lu	da134a2eb7	minor	2020-12-22 17:43:13 -08:00
Chris Lu	beb3b8ddb0	shell: change bucket.list to s3.bucket.list, same for create and delete	2020-12-22 17:40:55 -08:00
Chris Lu	4cf3176cbb	filer first, master second	2020-12-22 17:34:44 -08:00
Chris Lu	a1009e8044	filer: add -defaultStoreDir so that filer.toml can be skipped fix https://github.com/chrislusf/seaweedfs/issues/1659	2020-12-22 17:33:40 -08:00
Chris Lu	a0990b929d	minor	2020-12-22 17:33:40 -08:00
Baptiste Mille-Mathias	0ed5345954	[shell] Add more information to bucket.create State the replication setting that will be honor if not defined.	2020-12-22 17:18:59 +01:00
Chris Lu	97e3432dfe	avoid wrong error fix https://github.com/chrislusf/seaweedfs/issues/1691	2020-12-22 02:46:24 -08:00
Chris Lu	4f31c1bb94	go fmt	2020-12-22 02:34:08 -08:00
Chris Lu	6c4f32d173	remove unused code	2020-12-22 02:33:31 -08:00
Chris Lu	90d785a15f	filer: redis, redis cluster, cassandra support super large directory	2020-12-22 02:26:05 -08:00
Chris Lu	fe46411cd4	handle directory listing edge cases	2020-12-21 23:37:43 -08:00
Chris Lu	c74bede730	handle listing directories	2020-12-21 23:19:05 -08:00
Chris Lu	adf8cb4000	filer: path-specific stores trim out common prefixes	2020-12-21 22:57:13 -08:00
Chris Lu	0823bde8d8	refactor: split into two files	2020-12-21 22:05:15 -08:00
Chris Lu	8ab70ccf76	refactor	2020-12-21 21:46:32 -08:00
Chris Lu	65cc6dc636	refactor to dedicated function	2020-12-21 21:18:34 -08:00
Chris Lu	488c2680e8	change default filer store root	2020-12-21 21:15:22 -08:00
Chris Lu	f5a19a87be	filer: report error if the path specific store is not found related https://github.com/chrislusf/seaweedfs/issues/1691	2020-12-21 12:43:52 -08:00
Chris Lu	6912bf94ae	2.16	2020-12-20 15:30:12 -08:00
Chris Lu	41c0f3ad24	filer: support path-specific filer store	2020-12-19 01:27:09 -08:00
Chris Lu	2260864b45	a cleaner fix	2020-12-18 10:39:30 -08:00
Chris Lu	b7fef8f693	fix when a volume server has only hdd volumes	2020-12-18 10:34:16 -08:00
Chris Lu	0d5683fb0e	todo: load path-specific store from filer.toml	2020-12-18 03:05:42 -08:00
Chris Lu	3269fd7eaf	filer: use store by path	2020-12-18 02:57:49 -08:00
Chris Lu	e605f1e001	refactor, add getActualStore() by path	2020-12-18 02:55:00 -08:00
Chris Lu	23903aa95b	refactoring	2020-12-18 02:35:45 -08:00
Chris Lu	4e6b316913	assert FilerStoreWrapper is VirtualFilerStore	2020-12-18 02:17:06 -08:00
Chris Lu	9abf016af1	Merge branch 'master' into support_ssd_volume	2020-12-17 22:30:19 -08:00
Chris Lu	738c4fd203	fix logging when error address https://github.com/chrislusf/seaweedfs/pull/1685#issuecomment-747845601	2020-12-17 19:37:46 -08:00
Chris Lu	d9e8479c06	adjust UI max count	2020-12-17 13:47:51 -08:00
Chris Lu	3cdf5945a2	adjust UI	2020-12-17 13:37:00 -08:00
Chris Lu	f696a2b2a7	assign volumes based on disk type	2020-12-17 13:25:05 -08:00
Chris Lu	daa8157fc2	Merge branch 'master' into support_ssd_volume	2020-12-17 13:05:20 -08:00
Chris Lu	e2076201d7	volume: avoid reprocessing the same volume fix https://github.com/chrislusf/seaweedfs/issues/1682	2020-12-17 13:03:39 -08:00
Chris Lu	986cbdf7d9	Revert "Merge pull request #1683 from qieqieplus/master" This reverts commit `8cb67952db`, reversing changes made to `200e56215a`.	2020-12-17 12:46:42 -08:00
Chris Lu	f56e6d231e	Revert "Merge pull request #1683 from qieqieplus/master" This reverts commit `8cb67952db`, reversing changes made to `200e56215a`.	2020-12-17 12:46:20 -08:00
Chris Lu	1bf22c0b5b	go fmt	2020-12-16 09:14:05 -08:00
Chris Lu	1d88865869	passing disk type along	2020-12-16 09:10:14 -08:00
Chris Lu	23014b6810	Merge branch 'master' into support_ssd_volume	2020-12-16 08:26:51 -08:00
Chris Lu	b789767cd1	filer: return http status 499 for client closed connection fix https://github.com/chrislusf/seaweedfs/issues/1684	2020-12-16 08:18:00 -08:00
chenqieqie	45f902a9b7	smaller critical section	2020-12-16 20:23:51 +08:00
chenqieqie	4e58a4f24e	fix race condition when loading volumes concurrently	2020-12-16 18:49:10 +08:00
Chris Lu	3c7f9633eb	show error a little bit more obviously fix https://github.com/chrislusf/seaweedfs/issues/1678	2020-12-14 09:39:33 -08:00
Chris Lu	97dcae827b	Merge branch 'master' into support_ssd_volume	2020-12-14 01:27:31 -08:00
Chris Lu	3d47c38262	collection.delete requires _default_ as the default empty collection name fix https://github.com/chrislusf/seaweedfs/issues/1677	2020-12-14 01:05:20 -08:00
Chris Lu	2e8dba571b	adjust volume server UI	2020-12-14 00:51:57 -08:00
Chris Lu	bf5873022d	Merge branch 'master' into support_ssd_volume	2020-12-14 00:32:17 -08:00
Chris Lu	b7e3ca9172	Merge pull request #1650 from kmlebedev/masterServerEnableUI security master use access.ui	2020-12-14 00:31:51 -08:00
Chris Lu	021358749b	better detect IP address following advice from https://github.com/chrislusf/seaweedfs/issues/1671	2020-12-14 00:30:20 -08:00
Konstantin Lebedev	c4459249ef	ui interface is always on	2020-12-14 13:24:41 +05:00
Chris Lu	5cffddebb9	add logging	2020-12-14 00:11:52 -08:00
Chris Lu	933f2f4cfd	find location with matching disk type	2020-12-13 23:39:00 -08:00
Chris Lu	94525aa0fd	allocate volume by disk type	2020-12-13 23:08:21 -08:00
Chris Lu	f6a419c26c	disk type configurable for each folder	2020-12-13 22:49:56 -08:00
Chris Lu	7c52a35942	refactor	2020-12-13 22:29:52 -08:00
Chris Lu	c45d6a865b	Merge branch 'master' into support_ssd_volume	2020-12-13 22:14:18 -08:00
Chris Lu	38fc89d041	filer store: sql update if any insert error happens fix https://github.com/chrislusf/seaweedfs/issues/1673	2020-12-13 20:49:44 -08:00
Chris Lu	23280257df	sql put kv: avoid unnecessary update	2020-12-13 19:45:47 -08:00
Chris Lu	a9db24cd05	master allocate volumes if ssd type runs out	2020-12-13 19:44:57 -08:00
Chris Lu	ac4d4a65af	Update Makefile	2020-12-13 12:34:58 -08:00
Chris Lu	8baba93fce	rename parameter name to "disk"	2020-12-13 12:06:21 -08:00
Chris Lu	51eadaf2b6	rename parameter name to "disk"	2020-12-13 12:05:31 -08:00
Chris Lu	0d2ec832e2	rename from volumeType to diskType	2020-12-13 11:59:32 -08:00
Chris Lu	715b199eeb	fix tests	2020-12-13 04:14:50 -08:00
Chris Lu	66cd219b15	Merge branch 'master' into support_ssd_volume	2020-12-13 04:08:36 -08:00
Chris Lu	813453f3e1	2.15	2020-12-13 03:45:35 -08:00
Chris Lu	17372ac3da	balance ssd volumes	2020-12-13 03:40:33 -08:00
Chris Lu	d156c74ec0	volume server set volume type and heartbeat to the master	2020-12-13 03:11:24 -08:00
Chris Lu	e9cd798bd3	adding volume type	2020-12-13 00:58:58 -08:00
Chris Lu	16cd6fb278	fix variable name	2020-12-12 21:19:40 -08:00
Chris Lu	5c465293e9	correctly determine whether a folder is empty avoid edge cases that deleting the folder if previous 32 directories are all empty early terminate if one file is found	2020-12-12 16:19:29 -08:00
Chris Lu	14910d035c	the pagination size was too big for recursive deletion	2020-12-12 13:26:10 -08:00
Chris Lu	f930c713fc	more efficient recursion	2020-12-12 13:25:19 -08:00
Chris Lu	316d1b4e69	refactor APIs	2020-12-12 12:42:53 -08:00
Chris Lu	03637d6f57	s3: move "delete-directory-if-empty" to read time move "delete-directory-if-empty" to read time instead of entry deletion time the listing speed for a s3 bucket folder will slow down if it has many sub folders related to `0d345ac97d` fix https://github.com/chrislusf/seaweedfs/issues/1647 fix https://github.com/chrislusf/seaweedfs/issues/1670	2020-12-12 03:38:34 -08:00
Chris Lu	37075a414d	adjust logs	2020-12-12 03:33:57 -08:00
Chris Lu	ae655033ac	adjust logging	2020-12-11 16:57:53 -08:00
Chris Lu	d986c7196d	use append time instead of filer's own modification time fix https://github.com/chrislusf/seaweedfs/issues/1669	2020-12-11 16:55:18 -08:00
Chris Lu	3fedfec1e7	check cross device rename error	2020-12-10 23:50:32 -08:00
Chris Lu	c2f18a10cb	minor	2020-12-10 22:23:22 -08:00
Chris Lu	4b0c2a846b	add logs	2020-12-10 19:55:28 -08:00
Chris Lu	83078ac6ce	filer: change to /etc/seaweedfs folder on filer fix https://github.com/chrislusf/seaweedfs/issues/1666	2020-12-10 11:11:02 -08:00
Chris Lu	26731694f8	s3: use static configuration by default So that users can still use the previous configuration files. If leave it empty, s3 will try to use the version from filer	2020-12-10 00:59:04 -08:00
Chris Lu	765b3ef1be	save /etc/iam/identity.json inside filer store	2020-12-10 00:15:22 -08:00
Chris Lu	97c942b5ee	paginate through large directories	2020-12-09 23:23:38 -08:00
Chris Lu	02a160c3fd	s3: recursively iterate all sub folders fix https://github.com/chrislusf/seaweedfs/issues/1656	2020-12-09 16:47:34 -08:00
Chris Lu	a9c619c270	added error handling for saving	2020-12-09 01:34:24 -08:00
Chris Lu	4211601eab	set file handle reader to nil	2020-12-08 22:48:18 -08:00
Chris Lu	b52ae9cef8	add s3 debug	2020-12-08 22:27:04 -08:00
Chris Lu	8e78187a97	add back last read chunk cache to reader and properly close the reader	2020-12-08 22:26:46 -08:00
Chris Lu	900d22c6ec	mount: avoid memory leaking read buffer fix https://github.com/chrislusf/seaweedfs/issues/1654 the reader goes together with the file handle, which may stay for a long time.	2020-12-08 02:38:53 -08:00
Chris Lu	06bb7bf6c0	adding /usr/local/etc/seaweedfs for bsd style config search path related to https://reviews.freebsd.org/D27391	2020-12-07 16:46:48 -08:00
Chris Lu	fce8803087	break import cycle	2020-12-07 00:29:17 -08:00
Chris Lu	606051cc79	filer: cache "/etc" filder small files to filer store	2020-12-07 00:11:35 -08:00
Chris Lu	ac22f1dd26	filer subscribe: handle rename subscription	2020-12-07 00:10:49 -08:00
Chris Lu	eed87791b7	s3: subscribe to s3.configure changes	2020-12-07 00:10:29 -08:00
Chris Lu	01e2da5782	refactoring	2020-12-06 23:16:20 -08:00
Chris Lu	77286f8bea	fix test	2020-12-06 21:56:13 -08:00
Chris Lu	57578a6cd1	change parameter help message	2020-12-06 21:54:55 -08:00
Chris Lu	78f3ab439c	refactoring	2020-12-06 21:54:34 -08:00
Chris Lu	9acda432fe	fix import cycle	2020-12-06 20:12:52 -08:00
Chris Lu	ae5eb85a06	refactoring	2020-12-06 20:05:06 -08:00
Chris Lu	5931a2f53f	filer conf: support json conf only	2020-12-06 19:47:06 -08:00
Chris Lu	4c72482536	shell: fix fs.configure	2020-12-06 19:43:31 -08:00
Chris Lu	46b91228af	2.14	2020-12-06 18:56:58 -08:00
Chris Lu	263eb29e9f	filer: add option to cache small files to filer store	2020-12-04 22:39:43 -08:00
Chris Lu	d171d9f988	volume: ensure the volume dat and idx files are always closed fix https://github.com/chrislusf/seaweedfs/issues/1646	2020-12-04 21:50:26 -08:00
Konstantin Lebedev	eadbba5c95	security master use access.ui	2020-12-04 12:57:59 +05:00
Chris Lu	cc839f935d	better unit detection	2020-12-03 19:23:59 -08:00
Chris Lu	b219ccfe68	Update volume_ttl_test.go	2020-12-03 12:55:44 -08:00
Chris Lu	a83b8e7b01	ttl fix: 3y was converted into 96m	2020-12-03 10:53:29 -08:00
Chris Lu	5fb60b7135	Merge pull request #1596 from kmlebedev/store_s3cred S3 credentials store in filer	2020-12-03 00:40:20 -08:00
Konstantin Lebedev	14699dfcef	use content field of entry	2020-12-02 17:19:05 +05:00
Konstantin Lebedev	a3d4b50a49	use entry content filed	2020-12-02 16:12:13 +05:00
Chris Lu	003b6245e7	fix nil	2020-12-02 00:09:19 -08:00
Chris Lu	f3bb645018	file open error	2020-12-01 23:37:49 -08:00
Chris Lu	dc0bc48257	return file open error	2020-12-01 23:36:49 -08:00
Chris Lu	45dba088cd	fix tests	2020-12-01 19:37:21 -08:00
Chris Lu	04062c56c7	webdav: improve webdav upload speed	2020-12-01 15:32:27 -08:00
Konstantin Lebedev	03620776ec	Merge branch 'upstream_master' into store_s3cred	2020-12-01 16:03:34 +05:00
Chris Lu	005a6123e9	Merge pull request #1643 from hilimd/master fix tls grpc ca path	2020-11-30 17:16:46 -08:00
Chris Lu	f4abd01adf	filer: cache small file to filer store	2020-11-30 04:34:04 -08:00
Chris Lu	a9c6be5fc3	set creation time	2020-11-30 03:11:52 -08:00
Chris Lu	141ce67c09	close http request body	2020-11-30 02:45:00 -08:00
Chris Lu	0e99531dbf	2.13	2020-11-29 17:00:03 -08:00
Chris Lu	965413c21b	shell: add volume.vacuum command	2020-11-28 23:18:02 -08:00
Chris Lu	96c48bc8a8	fix test	2020-11-28 03:03:43 -08:00
Chris Lu	c57a7839ec	adjust comments	2020-11-28 03:01:03 -08:00
Chris Lu	8438fb0cc3	fix ec shard spreading error	2020-11-28 02:21:16 -08:00
Chris Lu	536202e4e0	minor	2020-11-28 02:16:06 -08:00
Chris Lu	99c4e50d3d	minor	2020-11-28 00:14:11 -08:00
Chris Lu	2c913dde04	volume: detect and drop volumes with disk IO error from Jethro in slack: is it possible to make the assign request a bit smarter? Currently I’m in the state that a disk failed but all assign request are being send to this volume. It would be cool if the master sees this and stopped using this volume. e=HTTP(http://x:8089/913,045a782b63176edf) not 200 but 500 Internal Server Error Body={"size":740167,"error":"failed to write to local disk: write /mnt/v9/913.dat: input/output error","eTag":"ee4381e202212ff3aee647704c036689"} e=HTTP(http://x:8089/913,045a782c90240077) not 200 but 500 Internal Server Error Body={"size":792779,"error":"failed to write to local disk: write /mnt/v9/913.dat: input/output error","eTag":"c43463ccc11eb6eb2fc306f407a6a953"} e=HTTP(http://x:8089/913,045a782e6b7901ea) not 200 but 500 Internal Server Error Body={"size":3962392,"error":"failed to write to local disk: write /mnt/v9/913.dat: input/output error","eTag":"04c91198e9b276c81f11dbf189af5d28"}	2020-11-28 00:09:29 -08:00
Chris Lu	9ac4935f22	read from volume index file directly instead of open a separate file fix https://github.com/chrislusf/seaweedfs/issues/1640 read from volume index file directly instead of open a separate file, to ensure reading latest index entries.	2020-11-27 16:18:48 -08:00
Chris Lu	85554bea38	filer: readonly handle static resources fix https://github.com/chrislusf/seaweedfs/issues/1641	2020-11-27 13:42:14 -08:00
Chris Lu	6d30b21b10	volume: add "-dir.idx" option for separate index storage fix https://github.com/chrislusf/seaweedfs/issues/1265	2020-11-27 03:17:10 -08:00
Chris Lu	3c229eb677	refactor	2020-11-26 17:21:55 -08:00
Chris Lu	c0fb4a3766	add comments	2020-11-26 15:22:42 -08:00
Chris Lu	6783ae5f8f	rename	2020-11-26 15:19:43 -08:00
Chris Lu	983fd90010	marked as WIP	2020-11-26 15:08:52 -08:00
Chris Lu	97f706f35b	just refactoring	2020-11-26 15:08:05 -08:00
Chris Lu	b11449f955	pass in dir.idx parameter to DiskLocation	2020-11-26 14:59:03 -08:00
Chris Lu	0d345ac97d	s3: remove empty parent folder on delete fix https://github.com/chrislusf/seaweedfs/issues/1637	2020-11-26 12:21:58 -08:00
Chris Lu	cc2bd97ad9	refactor	2020-11-26 11:25:56 -08:00
Chris Lu	0da7ecfd29	go fmt	2020-11-26 11:22:30 -08:00
Chris Lu	effa00ed08	refactor	2020-11-26 11:21:58 -08:00
Chris Lu	513bcd6e0d	filer: avoid duplicated FindEntry for deletion	2020-11-26 11:14:56 -08:00
Chris Lu	1ae108efca	add debug message	2020-11-26 10:41:10 -08:00
Chris Lu	a102157020	do not retry if failed with a non transport related error	2020-11-26 01:29:30 -08:00
Konstantin Lebedev	4e55baf5b1	s3 config changes	2020-11-26 03:50:53 +05:00
Konstantin Lebedev	52c8f2fc9a	s3iam test	2020-11-26 02:26:45 +05:00
Konstantin Lebedev	a26f1b2040	new pkg s3iam	2020-11-26 01:30:11 +05:00
Konstantin Lebedev	6206737df2	s3 configure	2020-11-25 21:02:31 +05:00
Chris Lu	eab53ea80d	filer leveldb store: a bit more efficient directory listing with prefix	2020-11-22 21:10:41 -08:00
Chris Lu	d3be2ceedd	2.12	2020-11-22 17:17:33 -08:00
Chris Lu	c7ebadc25d	avoid possible concurrent access inside ensureCorrectWritables()	2020-11-22 17:15:59 -08:00
hilimd	9aa990f80f	Merge pull request #39 from chrislusf/master sync	2020-11-22 20:29:17 +08:00
limd	e8296104fc	fix tls grpc ca path	2020-11-22 20:27:15 +08:00
Chris Lu	92f906b6fc	remove zstd fix https://github.com/chrislusf/seaweedfs/issues/1629	2020-11-21 13:06:45 -08:00
Aleksandr Demshin	176950ea78	fix typo	2020-11-20 19:50:46 +05:00
Chris Lu	7385103057	fix volume placement validating fix https://github.com/chrislusf/seaweedfs/issues/1626	2020-11-20 01:12:25 -08:00
Konstantin Lebedev	27e73de797	Merge branch 'upstream_master' into store_s3cred # Conflicts: # weed/s3api/filer_util.go	2020-11-19 18:16:44 +05:00
Chris Lu	290b5e2cd0	directly delete file chunks keeping current async deletions for now	2020-11-17 17:20:21 -08:00
Chris Lu	781585b195	remove unnecessary checking	2020-11-17 17:00:06 -08:00
Chris Lu	8cb8cd4cc5	add locks	2020-11-17 16:59:48 -08:00
Chris Lu	7c9e592c2d	s3: avoid nil resp when having error fix https://github.com/chrislusf/seaweedfs/issues/1622	2020-11-17 11:23:17 -08:00
Chris Lu	dc304342b2	fs.configure: configurable volume growth	2020-11-17 01:00:02 -08:00
Chris Lu	b04375a3d1	use alternative assign request	2020-11-17 00:36:21 -08:00
Chris Lu	9add554feb	fail fast if user configures collection for a bucket	2020-11-16 19:57:08 -08:00
Chris Lu	6204dc6e85	add comment	2020-11-16 16:57:31 -08:00
Chris Lu	ef3e068c51	deprecating "filer.options.buckets_fsync" in filer.toml	2020-11-16 16:56:49 -08:00
Chris Lu	5f19e81dab	filer confi: support hierachical configuration	2020-11-16 16:50:12 -08:00
Chris Lu	ed1ce3f299	adjust help message	2020-11-16 09:54:26 -08:00
Chris Lu	c0d279c54e	filere.conf: prefer to use json format	2020-11-15 21:48:17 -08:00
Chris Lu	83527a8f55	adds more help message	2020-11-15 21:31:14 -08:00
Chris Lu	af658ea970	fix typo	2020-11-15 21:16:28 -08:00
Chris Lu	98827d68d9	2.11	2020-11-15 20:25:56 -08:00
Chris Lu	ee2fa14dbe	filer conf: delete location specific configuration	2020-11-15 20:15:47 -08:00
Chris Lu	2bd6fd3bbe	remove unused function	2020-11-15 20:15:07 -08:00
Chris Lu	71056dae07	fs.configure: read and local add filer configuration	2020-11-15 18:09:35 -08:00
Chris Lu	0ea5c087ce	go fmt	2020-11-15 16:59:28 -08:00
Chris Lu	95c0de285d	refactoring	2020-11-15 16:58:48 -08:00
Chris Lu	500bcab953	refactoring	2020-11-15 14:41:56 -08:00
Chris Lu	590f02179d	filer: load filer conf when starting	2020-11-15 14:06:03 -08:00
Chris Lu	0a406f652e	load filer conf and match by prefix	2020-11-15 00:26:05 -08:00
Chris Lu	68043cfcac	add reference implementation to detect create/update/delete/rename events	2020-11-14 21:21:58 -08:00
Chris Lu	0fc9ffc603	able to read filer.conf if updated	2020-11-14 21:21:20 -08:00
Chris Lu	442e092995	filer watch "/etc" folder for configuration changes	2020-11-14 14:26:08 -08:00
ruitao.liu	a9990a1dc6	adjust check bucket if exist or has access.	2020-11-13 17:13:20 +08:00
Chris Lu	e6333da65a	enable admin to access all buckets	2020-11-12 13:57:54 -08:00
Chris Lu	7094492428	fix ErrNotFound	2020-11-12 13:30:46 -08:00
Chris Lu	c6a0704019	rename function	2020-11-12 13:30:08 -08:00
Chris Lu	ed3b0f81c6	Merge pull request #1613 from taozix/master check permission for bucket delete/head.	2020-11-12 13:25:28 -08:00
ruitao.liu	c4f0fd6e1b	skip if entry.Extended map is nil.	2020-11-12 17:59:31 +08:00
ruitao.liu	ab966410d2	return NoSuchBucket instead of InternalError delete non-existed bucket.	2020-11-12 16:44:16 +08:00
Chris Lu	a4dfa00030	adjust filer conf	2020-11-12 00:39:59 -08:00
Chris Lu	2251a8e472	minor	2020-11-12 00:38:58 -08:00
Chris Lu	5a16f17e47	remove unused message type	2020-11-12 00:38:23 -08:00
ruitao.liu	e06676f007	check permission for bucket delete/head.	2020-11-12 16:15:59 +08:00
Chris Lu	c32b6f4d9d	prepare for path based configuration	2020-11-11 23:46:45 -08:00
Chris Lu	0fbc27b640	Merge pull request #1612 from taozix/master add owner attr for bucket.	2020-11-11 22:19:47 -08:00
ruitao.liu	d7cc0498e0	check if bucket already exists.	2020-11-12 14:11:03 +08:00
Chris Lu	827930e249	Merge pull request #1610 from kmlebedev/prefer_read_in_this_dc prefer to read from volumes in this data center	2020-11-11 22:09:41 -08:00
Konstantin Lebedev	0880aff224	update dc flag desc	2020-11-12 10:07:52 +05:00
ruitao.liu	dbba8cb57e	fix if nil map in entry.	2020-11-12 11:50:19 +08:00
Konstantin Lebedev	aa30604cec	fix tests	2020-11-12 08:49:38 +05:00
Konstantin Lebedev	1eec5c8d5d	gen pb	2020-11-12 04:10:06 +05:00
Konstantin Lebedev	fc7baef5bb	fiil serverUrls sorted by data center	2020-11-12 02:13:33 +05:00
Chris Lu	745ee8d8f3	avoid unnecessary error message fix https://github.com/chrislusf/seaweedfs/issues/1611	2020-11-11 13:00:05 -08:00
Chris Lu	a29abd8600	add a {X-,}Idempotency-Key header for http POST fix https://github.com/chrislusf/seaweedfs/issues/1592 according to https://github.com/golang/go/issues/19943	2020-11-11 12:56:53 -08:00
Chris Lu	8708e7a64d	print out data size	2020-11-11 12:51:44 -08:00
Chris Lu	c6bd244ebd	add TODO	2020-11-11 12:51:27 -08:00
Konstantin Lebedev	b73ef6aa95	Merge remote-tracking branch 'upstream/master' into prefer_read_in_this_dc	2020-11-11 23:29:57 +05:00
Konstantin Lebedev	dc26012a3b	initial	2020-11-11 15:03:47 +05:00
Chris Lu	86cdb2a3e5	adjust logs	2020-11-11 02:01:24 -08:00
Chris Lu	73f934d5de	s3: do not close reader too early fix https://github.com/chrislusf/seaweedfs/issues/1609	2020-11-11 01:49:39 -08:00
ruitao.liu	5b636b3242	Add bucket owner attr.	2020-11-11 16:20:59 +08:00
Chris Lu	15956d8a8e	2.10	2020-11-10 23:17:13 -08:00
Chris Lu	de3bdd0651	delete old volume replica related to https://github.com/chrislusf/seaweedfs/issues/1607 old is: * older compaction revision * older modified time * smaller volume size	2020-11-10 12:26:05 -08:00
Chris Lu	1dd3a6ac36	avoid verbose logs	2020-11-09 15:56:11 -08:00
Chris Lu	6856b0d57e	filer: add API to add/modify/delete tagging	2020-11-09 01:00:07 -08:00
Konstantin Lebedev	6cd6ff6962	avoid blank response for 404	2020-11-04 22:54:47 +05:00
Chris Lu	0dafcf1f5a	volume: detect max volume count changes based on disk usage fix https://github.com/chrislusf/seaweedfs/issues/1594	2020-11-03 14:43:17 -08:00
Chris Lu	06ee199ef3	ensure entryViewCache is in sync with the chunks	2020-11-03 12:22:19 -08:00
Chris Lu	b132cb893f	in case some chunks modified earliar are added later	2020-11-03 12:22:19 -08:00
Chris Lu	0ae4d7bfdf	lock "addChunks()"	2020-11-03 12:22:19 -08:00
Konstantin Lebedev	e1190b3224	load S3 config from filer https://github.com/chrislusf/seaweedfs/issues/1500	2020-11-03 21:45:56 +05:00
Chris Lu	d29b787220	store together with "Seaweed-" prefix	2020-11-03 00:21:10 -08:00
Chris Lu	5448781dfc	filer: Headers with "Seaweed-" prefix are stored as extended properties	2020-11-03 00:15:51 -08:00
Chris Lu	0ecff9521d	avoid []byte reuse	2020-11-02 23:27:54 -08:00
Chris Lu	031aa70da6	Revert "revert bytebufferpool, seems problematic" This reverts commit `e0fcab47c0`.	2020-11-02 19:11:25 -08:00
Chris Lu	7d5db73d98	go fmt	2020-11-02 14:21:07 -08:00
Chris Lu	a9662d44ac	avoid closing chunkSaveErrChan too early	2020-11-02 14:20:38 -08:00
Chris Lu	e0fcab47c0	revert bytebufferpool, seems problematic	2020-11-02 13:49:38 -08:00
Chris Lu	e1423a83dd	mount: clear possible leftover entry data in memory	2020-11-01 23:28:56 -08:00
Chris Lu	9b10d1ac33	2.08	2020-11-01 12:45:20 -08:00
Chris Lu	8750cac090	move to util.RetryWaitTime	2020-11-01 02:36:43 -08:00
Chris Lu	ef908e166b	break import cycle	2020-11-01 02:28:11 -08:00
Chris Lu	4609644658	mount: remove files that may still are open related to https://github.com/chrislusf/seaweedfs/issues/1581	2020-11-01 01:33:26 -08:00
Chris Lu	df8d976bb0	refactoring	2020-11-01 01:58:48 -07:00
Chris Lu	7192a378cc	add retry to master clients also	2020-11-01 01:40:16 -07:00
Chris Lu	004aa9ca5d	only retry transport related errors	2020-11-01 01:21:44 -07:00
Chris Lu	f7b21973ac	mount: add retry for all operations with filer fix https://github.com/chrislusf/seaweedfs/issues/1589	2020-11-01 01:11:08 -07:00
Chris Lu	de86945aeb	go fmt	2020-10-31 16:45:38 -07:00
Chris Lu	306062b4e7	filer: add CORS support	2020-10-31 16:44:03 -07:00
Chris Lu	05acc33294	volume: add CORS support	2020-10-31 16:31:39 -07:00
Chris Lu	9708df47d1	ensure offset in the index file is positive	2020-10-31 15:50:01 -07:00
Chris Lu	10f9f1cc26	handle mknod requests	2020-10-30 23:51:32 -07:00
Chris Lu	8826601be1	mount: optional limit for the number of concurrent writers	2020-10-30 21:22:20 -07:00
Chris Lu	be95f68ca7	scaffold for sftpd	2020-10-30 20:57:08 -07:00
Konstantin Lebedev	d2a4ffdd60	glog	2020-10-31 03:21:54 +05:00
Konstantin Lebedev	46303c36bf	When the volume server is stopped, the master server immediately sees the deletion of volumes	2020-10-31 02:48:25 +05:00
Chris Lu	eb32af25d4	make space for the QR code	2020-10-30 10:55:31 -07:00
Chris Lu	7ecbb4b3c9	clean up	2020-10-30 10:46:31 -07:00
Kenny	9cc6575ab5	Update filer_copy.go 修改 weed filer.copy 上传目录时文件名变子目录的问题	2020-10-30 18:16:49 +08:00
Chris Lu	c057dd1f64	add back upload retries	2020-10-30 02:16:34 -07:00
Chris Lu	19098f2c2d	make a copy of the filer.entry for most read operations	2020-10-30 01:23:26 -07:00
Chris Lu	6135cbaa02	clear file.entry	2020-10-30 01:23:26 -07:00
Chris Lu	02dc51b1ed	properly clear out the file.entry	2020-10-29 23:31:06 -07:00
Chris Lu	6560ac6466	volume loading: trim out unreachable idx file content fix https://github.com/chrislusf/seaweedfs/issues/1583	2020-10-29 22:25:23 -07:00
Chris Lu	0b68b68ec4	reload entry only when it is a hard link fix https://github.com/chrislusf/seaweedfs/issues/1581	2020-10-29 16:32:45 -07:00
Chris Lu	47047516d6	Revert "increase default volume file size limit to 1024" This reverts commit `0983060a` I am confused why this was changed to 1024 by myself. 1GB should be too large for most cases.	2020-10-29 15:46:29 -07:00
Chris Lu	6f8b426f4f	Merge pull request #1578 from taozix/master S3 bucket list, response with uploaded storageclass.	2020-10-29 08:32:27 -07:00
ruitao.liu	22a9ea0512	adjust s3 header file.	2020-10-29 16:05:40 +08:00
Chris Lu	c6d9974fe7	server: Add option to enable/disable volume server fix https://github.com/chrislusf/seaweedfs/issues/1572	2020-10-29 00:24:18 -07:00
ruitao.liu	b917be7955	S3 bucket list, response with uploaded storageclass.	2020-10-29 14:57:19 +08:00
Konstantin Lebedev	918546bbdb	add WaitForReady to grpc DialOption	2020-10-29 08:29:04 +05:00
Konstantin Lebedev	fab01f9d8d	add readonly	2020-10-28 23:17:03 +05:00
Konstantin Lebedev	884db215a1	add shell command volume mark writable	2020-10-28 22:47:09 +05:00
Konstantin Lebedev	1ad1b8c4f6	collection Volume ReadOnly Count with detailed status	2020-10-28 17:38:26 +05:00
Chris Lu	326fcdd86b	Merge pull request #1569 from taozix/master save s3 metadata to filer.	2020-10-28 03:27:24 -07:00
ruitao.liu	2dcc178d0a	save s3 metadata to filer.	2020-10-28 18:16:05 +08:00
Chris Lu	744c5594da	volume: automatically trim out unreachable entries	2020-10-28 01:14:39 -07:00
Chris Lu	3e925faddd	trim out name extension	2020-10-28 01:14:05 -07:00
shibinbin	a67eb1afa1	fix: restart volumeserver sometime occurred out of memory error	2020-10-28 12:03:06 +08:00
Chris Lu	53c3aad875	volume: add a note file to avoid incomplete volume files fix https://github.com/chrislusf/seaweedfs/issues/1567	2020-10-27 15:56:49 -07:00
Chris Lu	6da87720eb	shorter tag prefix	2020-10-27 15:33:26 -07:00
Chris Lu	24bf142596	copy large file first	2020-10-27 15:31:15 -07:00
Chris Lu	432c6f8db3	Merge pull request #1566 from taozix/master return x-amz-tag-count header when GET object.	2020-10-27 13:16:58 -07:00
Chris Lu	06c15ab35c	volume: add special handling for .dat larger than 32GB	2020-10-27 13:11:56 -07:00
ruitao.liu	72f6b77dea	fix tag prefix.	2020-10-27 18:01:37 +08:00
ruitao.liu	8766ca1b95	rename s3 tag prefix.	2020-10-27 17:33:24 +08:00
ruitao.liu	19026ae55d	return x-amz-tag-count header when GET object.	2020-10-27 16:49:31 +08:00
Chris Lu	44921220b0	2.07	2020-10-25 22:03:46 -07:00
Chris Lu	e71463a9eb	mount: invalide file cache when metadata is changed	2020-10-25 19:24:15 -07:00
Chris Lu	e219c57849	passing full path when assign volume locations	2020-10-25 15:46:29 -07:00
Chris Lu	f375b93aef	renaming	2020-10-25 15:32:43 -07:00
Chris Lu	63b0fb54f7	minor	2020-10-25 14:15:53 -07:00
Chris Lu	27ba0a2be3	2.06	2020-10-25 12:36:47 -07:00
Chris Lu	147d4d23b6	filer.copy: automatically use the right bucket	2020-10-25 11:21:33 -07:00
Chris Lu	91fd311f7a	fix logic to read entry or not	2020-10-25 10:33:51 -07:00
Chris Lu	10a4a628e9	refresh cached file entry from sync metadata updates	2020-10-25 09:42:50 -07:00
Konstantin Lebedev	2fb1fce8a0	avoid old values when missing read only flags	2020-10-25 16:21:40 +05:00
Chris Lu	da36abf033	go fmt	2020-10-24 20:12:04 -07:00
Chris Lu	16fae84414	fix format error	2020-10-24 20:11:31 -07:00
Chris Lu	b7b1eb51b1	default to 2 for better performance in common computer hardware	2020-10-24 20:08:26 -07:00
Chris Lu	79690b9cd8	revert changes causing wrong data	2020-10-24 20:07:19 -07:00
Chris Lu	9104cfa744	reduce locks	2020-10-24 19:40:35 -07:00
Chris Lu	e9d40b80b2	less verbose logs	2020-10-24 09:42:54 -07:00
Chris Lu	6c4fb243a7	remove limits to concurrent writers	2020-10-24 02:05:05 -07:00
Chris Lu	e0002f8dd7	check existing volumes for writable status	2020-10-24 01:34:31 -07:00
Chris Lu	19772d70d7	print for debugging	2020-10-24 00:12:02 -07:00
Chris Lu	5b92dfe302	adjust log level	2020-10-23 23:58:54 -07:00
Chris Lu	fb124f2d4a	conditionally display volume server public url link	2020-10-23 23:58:46 -07:00
Chris Lu	1b17f71939	adjust election timeout to 10 seconds	2020-10-23 23:06:44 -07:00
Chris Lu	ef9c32ea0d	adjust compression threshold	2020-10-23 23:05:55 -07:00
Chris Lu	0cea84e7ab	make it easy with setting up multiple masters in a clcuster	2020-10-23 18:18:46 -07:00
Chris Lu	fc689319ae	add volume public url	2020-10-23 18:03:23 -07:00
Chris Lu	71624858af	avoid reusing cached []byte this logic is just to ensure 100% correct. fuse message, data+message hearder > 512 will not be cached.	2020-10-23 11:31:57 -07:00
Chris Lu	29c9fa2ef2	byte buffer for uploading	2020-10-23 11:29:51 -07:00
Chris Lu	2579edbc60	fix bug found by tests	2020-10-23 10:05:40 -07:00
Chris Lu	e0c8507d93	memory pool to reduce memory allocation	2020-10-23 00:41:40 -07:00
Chris Lu	b81359823f	postgres: support empty user	2020-10-22 14:27:47 -07:00
Chris Lu	11716fbf6f	make reading error more obvious	2020-10-22 13:33:45 -07:00
Chris Lu	d75bc62196	to read files that are quickly changing	2020-10-22 12:59:15 -07:00
Chris Lu	575d7952a1	add available resource stats fix https://github.com/chrislusf/seaweedfs/issues/1555	2020-10-22 09:13:47 -07:00
Chris Lu	5179e559f7	skip empty logs	2020-10-22 00:35:48 -07:00
Chris Lu	de044e466e	Merge pull request #1553 from kmlebedev/multi_domainname Multi domainname	2020-10-21 23:56:03 -07:00
Chris Lu	e73d6c9526	able to print partial file id address https://github.com/chrislusf/seaweedfs/issues/1552	2020-10-21 23:48:07 -07:00
Konstantin Lebedev	a9a7005687	set desc of option	2020-10-22 11:23:00 +05:00
Chris Lu	720b1d9b88	adding locking to avoid nil VolumeLocationList fix panic: runtime error: invalid memory address or nil pointer dereference Oct 22 00:53:44 bedb-master1 weed[8055]: [signal SIGSEGV: segmentation violation code=0x1 addr=0x8 pc=0x17658da] Oct 22 00:53:44 bedb-master1 weed[8055]: goroutine 310 [running]: Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(VolumeLocationList).Length(...) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/volume_location_list.go:35 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(VolumeLayout).enoughCopies(...) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/volume_layout.go:376 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(VolumeLayout).ensureCorrectWritables(0xc000111d50, 0xc000b55438) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/volume_layout.go:202 +0x5a Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(Topology).SyncDataNodeRegistration(0xc00042ac60, 0xc001454d30, 0x1, 0x1, 0xc0005fc000, 0xc00135de40, 0x4, 0xc00135de50, 0x10, 0x10d, ...) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/topology.go:224 +0x616 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/server.(MasterServer).SendHeartbeat(0xc000162700, 0x23b97c0, 0xc000ae2c90, 0x0, 0x0) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/server/master_grpc_server.go:106 +0x325 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/pb/master_pb._Seaweed_SendHeartbeat_Handler(0x1f8e7c0, 0xc000162700, 0x23b0a60, 0xc00024b440, 0x3172c38, 0xc000ab7100) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/pb/master_pb/master.pb.go:4250 +0xad Oct 22 00:53:44 bedb-master1 weed[8055]: google.golang.org/grpc.(Server).processStreamingRPC(0xc0001f31e0, 0x23bb800, 0xc000ac5500, 0xc000ab7100, 0xc0001fea80, 0x311fec0, 0x0, 0x0, 0x0) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:1329 +0xcd8 Oct 22 00:53:44 bedb-master1 weed[8055]: google.golang.org/grpc.(Server).handleStream(0xc0001f31e0, 0x23bb800, 0xc000ac5500, 0xc000ab7100, 0x0) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:1409 +0xc5c Oct 22 00:53:44 bedb-master1 weed[8055]: google.golang.org/grpc.(Server).serveStreams.func1.1(0xc0001ce8b0, 0xc0001f31e0, 0x23bb800, 0xc000ac5500, 0xc000ab7100) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:746 +0xa5 Oct 22 00:53:44 bedb-master1 weed[8055]: created by google.golang.org/grpc.(*Server).serveStreams.func1 Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:744 +0xa5 Oct 22 00:53:44 bedb-master1 systemd[1]: weedmaster.service: Main process exited, code=exited, status=2/INVALIDARGUMENT Oct 22 00:53:44 bedb-master1 systemd[1]: weedmaster.service: Failed with result 'exit-code'.	2020-10-21 23:15:48 -07:00
Chris Lu	2bc05cfbc8	fix typo	2020-10-21 22:26:41 -07:00
Chris Lu	4ff2ceee33	UI fix on rendering EC volumes addressing UI problem with https://github.com/chrislusf/seaweedfs/issues/1551	2020-10-21 22:05:58 -07:00
Chris Lu	f5ee19e0db	shortcut for appending operations	2020-10-21 20:44:01 -07:00
Chris Lu	b5917bf846	clean up	2020-10-21 19:31:26 -07:00
Chris Lu	9884dfc369	use bytebuffer for interval list	2020-10-21 19:29:51 -07:00
Chris Lu	bd103c143a	add lock for vidCache	2020-10-21 19:28:59 -07:00
Konstantin Lebedev	05b5f12f2e	multiplate DomainNames through comma	2020-10-21 20:48:51 +05:00
Chris Lu	81cf8d04df	minor	2020-10-21 02:42:23 -07:00
Chris Lu	d90e7ee3cc	adjust log level	2020-10-21 02:17:40 -07:00
Chris Lu	3bf0116de1	mount: less channel waiting	2020-10-21 02:16:21 -07:00
Chris Lu	c31b254248	mount: shortcut when there is only one chunk	2020-10-20 23:48:29 -07:00
Chris Lu	f64252023e	Revert "a better byte buffer pool" This reverts commit `59c6422777`.	2020-10-20 23:21:54 -07:00
Chris Lu	ad652ffb97	avoid close closed channel	2020-10-20 22:54:34 -07:00
Chris Lu	20442dc1bc	avoid extra data copying	2020-10-20 22:54:21 -07:00
Chris Lu	59c6422777	a better byte buffer pool	2020-10-20 22:53:58 -07:00
Chris Lu	6c92fa349d	filer: support default rack fix https://github.com/chrislusf/seaweedfs/issues/1546	2020-10-20 17:41:39 -07:00
Chris Lu	b3aa2fab9a	s3: report error to s3 when updating an object but it is already a directory fix https://github.com/chrislusf/seaweedfs/issues/1545	2020-10-20 10:25:16 -07:00
Chris Lu	410b818aa7	master: avoid timer leakage	2020-10-19 14:24:57 -07:00
Chris Lu	69f0da5d73	mount: pass along replication and collection parameters	2020-10-19 00:12:42 -07:00
Chris Lu	c0ab458671	report nil instead of EOF for empty files related to https://github.com/chrislusf/seaweedfs/issues/1541	2020-10-17 11:03:46 -07:00
Chris Lu	898db14729	2.05	2020-10-17 02:42:33 -07:00
Chris Lu	09bab17aff	mount: avoid "send on closed channel"	2020-10-16 12:54:23 -07:00
Chris Lu	9d80a3428c	add debug_webdav	2020-10-16 12:53:56 -07:00
Chris Lu	37f165d743	webdav: return io.EOF when at end of a file fix https://github.com/chrislusf/seaweedfs/issues/1344	2020-10-16 12:53:37 -07:00
Chris Lu	68d39c86f1	mysql, postgres, cassandra: change kv key to base64 encoding The exisitng key-value operation for stores using mysql, postgres, and maybe cassandra are already broken. The kv is used to store hardlink, filer store signature and replication progress. So users using hardlink and also uses mysql, postgres, or cassandra will have broken hard links. Users using filer.sync will need to re-sync the files.	2020-10-16 11:10:12 -07:00
Chris Lu	ee1fc6558a	refactor	2020-10-16 11:02:44 -07:00
Chris Lu	06e6341097	set initial write time to avoid race condition with delayed chunks	2020-10-15 11:08:45 -07:00
Chris Lu	d598c47136	log errors	2020-10-15 11:01:43 -07:00
Chris Lu	5039aff310	release the goroutine to read errors	2020-10-15 10:59:18 -07:00
Chris Lu	ace0ea3d28	s3: avoid duplicated bucket	2020-10-15 10:52:20 -07:00
Konstantin Lebedev	dc2e13092d	add number of read only volumes metric	2020-10-15 15:48:40 +05:00
Chris Lu	e714c28a02	mount: async write file chunk	2020-10-14 23:28:03 -07:00
Chris Lu	c95d3d9dff	adjust tests	2020-10-14 12:27:52 -07:00
Chris Lu	93bcf56514	file read report EOF fix https://github.com/chrislusf/seaweedfs/issues/1344	2020-10-14 12:18:24 -07:00
Chris Lu	1069b325dd	shell: volumeServer.evacuate adds printout for ec volumes	2020-10-13 20:26:03 -07:00
Chris Lu	0542911e29	go fmt	2020-10-13 19:50:46 -07:00
Chris Lu	58fa506491	minor	2020-10-13 19:50:22 -07:00
Chris Lu	28d4e1a51b	mount: retry for directory listing with filer related to https://github.com/chrislusf/seaweedfs/issues/1530	2020-10-13 19:49:52 -07:00
Chris Lu	c127da1219	filer: linearize timeout for large chunk of data	2020-10-13 14:04:46 -07:00
Chris Lu	aac4cb1f0c	adds errror on read and write	2020-10-13 13:53:34 -07:00
Chris Lu	9b4f7fed14	mount: report filer IO error related to https://github.com/chrislusf/seaweedfs/issues/1530	2020-10-13 11:21:13 -07:00
Chris Lu	3f7d1d1bf1	Only wait on retryable requests	2020-10-13 00:29:46 -07:00
Chris Lu	b18f21cce1	mount: fix bound tree with filer.path fix https://github.com/chrislusf/seaweedfs/issues/1528	2020-10-12 21:58:37 -07:00
Chris Lu	f022aff289	add back http.StatusPartialContent revert `e7c04af1d0`	2020-10-12 12:26:25 -07:00
Chris Lu	bbd0afd37e	2.04	2020-10-11 21:25:30 -07:00
Chris Lu	7704469d29	mount: outsideContainerClusterMode changed to use volume server publicUrl	2020-10-11 20:42:15 -07:00
Chris Lu	723ae11db4	refactoring in order to adjust volume server url later	2020-10-11 20:15:10 -07:00
Chris Lu	d155f907c2	mount: configurable read wait time	2020-10-10 20:09:43 -07:00
Chris Lu	9b0e8ef026	filer: added QR code to transfer files with mobile devices	2020-10-10 18:00:21 -07:00
Chris Lu	8a52379ecb	add retry if volume can not be found	2020-10-10 16:02:39 -07:00
Chris Lu	70af0ec24c	filer: fix hanging on read	2020-10-10 16:02:10 -07:00
Chris Lu	cff8bb6554	return proper error	2020-10-10 15:43:22 -07:00
Chris Lu	6da8eef54f	fix logic error	2020-10-09 00:01:47 -07:00
Chris Lu	5d01dd28a6	add logging	2020-10-08 23:38:00 -07:00
Chris Lu	8d34eb0050	mount:exponentially backoff if read error for about 10 minutes	2020-10-08 23:31:26 -07:00
Chris Lu	6e1f936efd	refactoring	2020-10-08 23:19:42 -07:00
Chris Lu	b2ee5873fb	fix error not being returned	2020-10-08 23:19:20 -07:00
Chris Lu	b067853162	mount: ignore recursion error fix https://github.com/chrislusf/seaweedfs/issues/1514 the host OS should manage recursive deletion.	2020-10-08 21:55:55 -07:00
Chris Lu	e1a8e3f900	fix logging	2020-10-08 18:40:38 -07:00
Chris Lu	8676db0f6a	adjust logging	2020-10-08 18:33:37 -07:00
Chris Lu	bd8b9b0c2e	some changes to prepare fix after failed reads	2020-10-08 18:33:06 -07:00
Chris Lu	78859e35fc	s3: adjust config examples	2020-10-08 10:12:09 -07:00
Chris Lu	9e7a2772b1	s3: only admin can list all buckets	2020-10-08 10:12:09 -07:00
Chris Lu	eed492b73b	randomize file locations	2020-10-07 23:58:32 -07:00
Chris Lu	ec08a1670b	volume: avoid hanging when stopping a volume server	2020-10-07 23:48:24 -07:00
Chris Lu	62d3d3aea0	go fmt	2020-10-07 23:30:54 -07:00
Chris Lu	e91b9c85a2	s3: Added support for "List" action in weed s3 -config=... in the config file. fix https://github.com/chrislusf/seaweedfs/issues/1511	2020-10-07 23:22:35 -07:00
Chris Lu	a8624c2e4f	read from alternative replica related to https://github.com/chrislusf/seaweedfs/issues/1512	2020-10-07 22:49:04 -07:00
Chris Lu	da4edf3651	master: check peers for existing leader before starting a leader election fix https://github.com/chrislusf/seaweedfs/issues/1509	2020-10-07 01:25:39 -07:00
Chris Lu	c543762e23	volume: auto expands -max setting I have tried to run weed volume but got errors... cmd: weed volume -max=0 -mserver="127.0.0.1:9333" -port=8080 -dir="/var/www/d1/sfs,/var/www/d2/sfs,/var/www/d3/sfs" log: I1006 15:09:26 17040 file_util.go:23] Folder /var/www/d1/sfs Permission: -rwxr-xr-x I1006 15:09:26 17040 file_util.go:23] Folder /var/www/d2/sfs Permission: -rwxr-xr-x I1006 15:09:26 17040 file_util.go:23] Folder /var/www/d3/sfs Permission: -rwxr-xr-x F1006 15:09:26 17040 volume.go:142] 3 directories by -dir, but only 1 max is set by -max What am I doing wrong? Thx.	2020-10-06 09:05:30 -07:00
Chris Lu	36492c47ec	adjust	2020-10-05 14:06:18 -07:00
Chris Lu	8f8e9ddbde	adjust API	2020-10-05 09:47:07 -07:00
Konstantin Lebedev	e4f2d9eb4a	We return etag using the same algorithm as aws s3 https://teppen.io/2018/06/23/aws_s3_etags/	2020-10-05 14:43:32 +05:00
Chris Lu	4fc673341f	2.03	2020-10-04 21:58:22 -07:00
Chris Lu	6b591b02af	adjust log level	2020-10-04 16:21:43 -07:00
Устюжанин Антон Александрович	702b1cb876	fix: remove deleted peers if resumeState = true	2020-10-04 21:56:17 +05:00
Chris Lu	d522df0ecd	adjust log level	2020-10-04 01:35:32 -07:00
Chris Lu	666859f7cd	mount: read prefetching	2020-10-04 01:31:04 -07:00
Chris Lu	bc47835997	mount: avoid duplicated reads	2020-10-03 20:16:42 -07:00
Chris Lu	8d65ad1444	Revert "mount: adds read prefetching" This reverts commit `692f0614d1`.	2020-10-03 19:40:56 -07:00
Chris Lu	692f0614d1	mount: adds read prefetching	2020-10-03 16:50:09 -07:00
Chris Lu	707936f482	re-enable caching larger than 16MB revert `62ce85610e`	2020-10-03 14:12:38 -07:00
Chris Lu	2c20ef72ae	fix starting problem	2020-10-03 13:56:38 -07:00
Chris Lu	d77e3c96e9	FUSE: use the mount directory permissions	2020-10-03 13:37:35 -07:00
Chris Lu	95ffed87c0	Merge pull request #1505 from ustuzhanin/remove_raft_state Resume raft state	2020-10-03 10:38:47 -07:00
Устюжанин Антон Александрович	6fbfe73683	fix: resumeState	2020-10-03 22:16:47 +05:00
Устюжанин Антон Александрович	dc31b19469	fix: restore raft state	2020-10-03 14:03:41 +05:00
Chris Lu	f781cce500	s3: support object tagging * GetObjectTagging * PutObjectTagging * DeleteObjectTagging	2020-10-02 22:21:51 -07:00
Устюжанин Антон Александрович	8c82fb7e5f	fix: restore raft state	2020-10-02 23:01:20 +05:00
Chris Lu	9ab98fa912	s3 metrics adjust the label	2020-10-01 07:17:47 -07:00
Chris Lu	f1e879fe7a	2.02	2020-10-01 07:15:49 -07:00
Chris Lu	a1c01d716b	volume: avoid deadlock when deleting volumes fix https://github.com/chrislusf/seaweedfs/issues/1501	2020-10-01 07:10:03 -07:00
Konstantin Lebedev	86329bbf2b	label name is statusCode	2020-10-01 01:22:38 +05:00
Konstantin Lebedev	68463e92c1	add status code in S3RequestCounter	2020-10-01 00:59:39 +05:00
Chris Lu	79ab10e300	adjust help message	2020-09-30 09:32:00 -07:00
Chris Lu	55cb68c09c	filer: replication follows master config if not specified	2020-09-30 09:15:55 -07:00
Chris Lu	b9887504e8	fix test	2020-09-27 23:19:50 -07:00
Chris Lu	f46eae284e	adjust for test	2020-09-27 23:08:11 -07:00
Chris Lu	75aca5e13d	2.01	2020-09-27 23:02:41 -07:00
Chris Lu	e6552b5e1e	filer: able to start s3 together	2020-09-27 23:00:43 -07:00
Chris Lu	474e2b6ac3	add a hard link marker to 16byte + maker, for future extensions	2020-09-27 22:38:30 -07:00
Chris Lu	c49e2bb9a3	adjust	2020-09-27 12:07:45 -07:00
Chris Lu	62ce85610e	skip caching too large chunks	2020-09-27 11:58:48 -07:00
Chris Lu	9ad2dcca2b	more tests	2020-09-27 11:42:51 -07:00
Chris Lu	e43d86c796	fix pre allocated volume size	2020-09-27 10:58:19 -07:00
Chris Lu	31fc7bb2e1	refactor adjust for faster test	2020-09-27 10:41:29 -07:00
Chris Lu	a37535cd9f	avoid non utf-8 in filename fix https://github.com/chrislusf/seaweedfs/issues/1493	2020-09-26 00:13:39 -07:00
Chris Lu	41be7a4c8f	filer: upload also set file size ensure works same as fuse mount	2020-09-25 12:02:06 -07:00
Chris Lu	35f6518c36	weed upload: usePublicUrl was not being used on big files fix https://github.com/chrislusf/seaweedfs/issues/1492	2020-09-25 02:41:38 -07:00
Chris Lu	22049dc2e6	Merge pull request #1491 from hilimd/master s3: fix get object metadata problem	2020-09-25 02:35:00 -07:00
limd	402aef8f30	s3: 1.fix spark reading S3 directory wildcard problem 2.fix the problem of the spark history service writing S3 directory reference git revsion number: `b41b7ea4d0`	2020-09-25 14:37:02 +08:00
Chris Lu	8516517c48	filer store: Cassandra supports username/password	2020-09-24 21:31:06 -07:00
limd	ed7816681a	Add cassandra authenticator mode	2020-09-25 11:22:47 +08:00
limd	370a98cf6b	code style	2020-09-25 11:19:42 +08:00
limd	aee3fd08b9	code style	2020-09-25 11:04:32 +08:00
Chris Lu	043b063136	Merge pull request #1482 from hilimd/master Fix: s3 delete object	2020-09-24 18:21:34 -07:00
limd	48c578410f	mount: rollback default value	2020-09-25 09:18:52 +08:00
Chris Lu	dbf5327b98	s3: handle response-content-encoding fix https://github.com/chrislusf/seaweedfs/issues/1487	2020-09-24 18:09:52 -07:00
Chris Lu	7726965a47	minor	2020-09-24 11:20:12 -07:00
Chris Lu	0790c6d605	fix empty hard link id	2020-09-24 11:16:43 -07:00
Chris Lu	1012df7bb5	switch hardlink id from int64 to bytes	2020-09-24 11:11:42 -07:00
Chris Lu	4856bce0ee	adjust for metrics port	2020-09-24 10:21:23 -07:00
Chris Lu	d2d3aec3e1	consolidate to one metricsPort in "weed server" mode	2020-09-24 09:55:02 -07:00
Chris Lu	2e9099369e	Merge pull request #1489 from kmlebedev/promhttp Promhttp	2020-09-24 09:50:48 -07:00
Chris Lu	2e7c361a0d	hardlink deletion factors in hardlink counter	2020-09-24 09:43:52 -07:00
Chris Lu	1295347958	adjust hardlink update simplify logic, pass entity content directly to hard link. The "weed mount" handles the logic to calculate hard link counter.	2020-09-24 09:43:00 -07:00
Konstantin Lebedev	98e9de6e11	fix style	2020-09-24 17:48:39 +05:00
Konstantin Lebedev	324e44d4b3	add start metrics server	2020-09-24 17:45:39 +05:00
limd	cbca14edc5	mount: fix k8s pvc and os mount directory permission bug	2020-09-24 18:07:16 +08:00
Chris Lu	5e239afdfc	hardlink works now	2020-09-24 03:06:48 -07:00
limd	59e91e9c7e	mount: fix k8s pvc mount directory permission	2020-09-24 10:25:45 +08:00
Chris Lu	c7d7b1a0f6	Merge pull request #1485 from LIBA-S/fix_oversized Correct the oversized state of volume after compaction	2020-09-23 19:24:30 -07:00
Chris Lu	b80a2b3cc9	Merge pull request #1486 from LIBA-S/fix_race_condition Fix a race condition when handle VolumeLocationList	2020-09-23 19:24:16 -07:00
LIBA-S	eecd6b5d35	Fix a race condition when handle VolumeLocationList	2020-09-23 20:56:51 +08:00
LIBA-S	0157798ebf	Correct the oversized state of volume after compaction	2020-09-23 20:27:42 +08:00
limd	9b21ec27e4	mount: auto created dir set corrct umask fix bug	2020-09-23 18:35:37 +08:00
hilimd	0db149fb5f	Merge pull request #18 from chrislusf/master sync	2020-09-23 17:32:05 +08:00
Chris Lu	f7a0ccb595	mount: auto created dir set corrct umask	2020-09-23 02:31:19 -07:00
Chris Lu	f0e325b6fc	mount: auto created directory follow umask	2020-09-23 02:27:57 -07:00
limd	d506080c36	rollback	2020-09-23 14:29:53 +08:00
limd	8f9f29b773	fixed the problem of empty directory when S3 deleted the directory	2020-09-23 13:33:13 +08:00
limd	1892677b22	fixed the problem of empty directory when S3 deleted the directory	2020-09-23 13:15:06 +08:00
hilimd	85570f27a0	Merge pull request #17 from chrislusf/master sync	2020-09-23 11:39:57 +08:00
limd	a99f63cb4d	1.add S3 copy directory function 2.fixed the problem of empty directory when S3 deleted the directory	2020-09-23 11:26:01 +08:00
Chris Lu	b61d33f251	expose only store wrapper to meta changes	2020-09-22 16:27:36 -07:00
Chris Lu	0adbb56cc1	rename	2020-09-22 16:24:13 -07:00
Chris Lu	3e52329cee	Revert "Merge pull request #1479 from LIBA-S/fix_oversized" This reverts commit `bd11f0b3e4`, reversing changes made to `ec5b9f1e91`.	2020-09-22 15:05:37 -07:00
Chris Lu	4a1fe4b8e2	add error logs	2020-09-22 09:16:10 -07:00
LIBA-S	8573ac82c6	Fix: remove the oversized state after compaction	2020-09-22 21:48:39 +08:00
James Hartig	658fc2e5b6	Allow option to enable volume pprof on server	2020-09-21 22:43:10 -04:00
James Hartig	91e4eca1e9	Fix deadlock with KeepConnected and SendHeartbeat There's the potential where we're writing to a clientConn and it goes away and we're stuck keeping a read lock on clientChansLock. This causes KeepConnected to not be able to remove the client since it requires a write lock on clientChansLock. This ends up backing up SendHeartbeat because it can't get a read lock.	2020-09-21 22:41:38 -04:00
Chris Lu	9cdbfc1a49	refactor	2020-09-21 17:34:38 -07:00
Chris Lu	63373a9f9f	filer: add file extended properties	2020-09-21 11:08:34 -07:00
Chris Lu	b9c1f3e9de	s3: fixes for list multipart upload	2020-09-21 10:51:24 -07:00
Chris Lu	289e62a305	master: better locking of in memory volume data related to https://github.com/chrislusf/seaweedfs/issues/1436#issuecomment-695880135	2020-09-20 23:07:55 -07:00
Chris Lu	9a3b564508	adjust printout	2020-09-20 16:01:56 -07:00
Chris Lu	62563a895a	refactoring	2020-09-20 16:00:01 -07:00
Chris Lu	c2faab23b6	refactor	2020-09-20 15:40:49 -07:00
Chris Lu	d013d09a9b	adjust logging	2020-09-20 15:38:59 -07:00
Chris Lu	f498c71199	shell: move volume operations to use flag parsing arguments	2020-09-20 09:27:34 -07:00
Chris Lu	fcbc520373	fix print out	2020-09-20 08:47:30 -07:00
Chris Lu	56094541c8	shell: adjust command option for collection.delete	2020-09-20 08:46:16 -07:00
Chris Lu	9cac9c4001	2.00	2020-09-20 00:47:04 -07:00
Chris Lu	dc4d2145ff	open up	2020-09-19 20:23:12 -07:00
Chris Lu	29abe980df	s3: add support for PostPolicy fix https://github.com/chrislusf/seaweedfs/issues/1426	2020-09-19 20:14:19 -07:00
Chris Lu	41d508edfd	go fmt	2020-09-19 14:10:26 -07:00
Chris Lu	5b40a2690a	refactoring	2020-09-19 14:09:58 -07:00
Chris Lu	2c21eb1971	volume: get metrics configuration from master fix https://github.com/chrislusf/seaweedfs/issues/1354	2020-09-19 00:03:00 -07:00
Chris Lu	2cbd1cf121	fix compilation	2020-09-18 00:15:54 -07:00
Chris Lu	23e9ede068	s3: collect metrics	2020-09-18 00:09:04 -07:00
Chris Lu	852e5f7cbc	filer: fix mongodb insert fix https://github.com/chrislusf/seaweedfs/issues/1471	2020-09-17 21:50:52 -07:00
Chris Lu	6544e60bea	s3 add metrics empty for now	2020-09-17 06:56:15 -07:00
Chris Lu	cb427d48fa	filer report metrics configuration	2020-09-17 06:46:51 -07:00
Chris Lu	e4e0234998	refactoring	2020-09-17 06:43:54 -07:00
Chris Lu	feca07bf96	Filer: ip bind fix https://github.com/chrislusf/seaweedfs/issues/1470	2020-09-16 10:26:08 -07:00
Chris Lu	e861a6a3ab	simplify metrics configuration logic	2020-09-16 01:39:30 -07:00
Chris Lu	be54eeb364	passing value by checkWithMaster() instead	2020-09-16 01:33:45 -07:00
Chris Lu	4d21de63ee	go fmt	2020-09-16 01:27:24 -07:00
Chris Lu	913a16268d	volume: load configuration from master at the start fix https://github.com/chrislusf/seaweedfs/issues/1469	2020-09-16 01:27:05 -07:00
Chris Lu	c9202c4b3d	add storage backend to GetMasterConfigurationResponse	2020-09-16 01:12:22 -07:00
Chris Lu	8a0710cb73	handle more than 2GB files related to https://github.com/chrislusf/seaweedfs/issues/1468	2020-09-16 00:41:50 -07:00
Chris Lu	854007bc98	minor	2020-09-16 00:37:57 -07:00
Chris Lu	10f9081526	filer: adjust meta data events to received timestamp if a client is already connected and start from t0. A message recieved at t+1 but with timestamp t-1 may not be processed by the client. This commit changes to the event received time, so the replication can be ordered.	2020-09-15 01:18:33 -07:00
Chris Lu	684a875876	update help message	2020-09-15 00:40:38 -07:00
Chris Lu	2d3a904a82	shell: volumeServer.evacuate adds option to skip non moveable volumes	2020-09-15 00:33:49 -07:00
Chris Lu	a595916342	shell: add volumeServer.evacuate command	2020-09-14 23:47:11 -07:00
Chris Lu	103fafe00b	weed export: print out [start,stop) content range	2020-09-14 22:57:23 -07:00
Chris Lu	ada996fe55	filer: support createing empty folder fix https://github.com/chrislusf/seaweedfs/issues/1161	2020-09-14 13:07:46 -07:00
Chris Lu	200fe5c83e	go fmt	2020-09-13 21:26:30 -07:00
Chris Lu	5d6753fb98	shell: add volumeServer.leave command	2020-09-13 21:25:51 -07:00
Chris Lu	1af95c5b76	refactoring	2020-09-13 12:41:26 -07:00
Chris Lu	f8fea19669	1.99	2020-09-12 13:46:33 -07:00
Chris Lu	b0c7de186d	filer: fix postgres prefixed directory listing problem fix https://github.com/chrislusf/seaweedfs/issues/1465	2020-09-12 13:37:03 -07:00
Chris Lu	1a7afe7e6a	adjust default value related to https://github.com/chrislusf/seaweedfs/issues/1453	2020-09-12 12:46:42 -07:00
Chris Lu	f2723c1bc8	do not idx file format revert `c9ab8d05fa`	2020-09-12 12:42:36 -07:00
Chris Lu	ba984a4e29	1.98	2020-09-12 04:13:02 -07:00
Chris Lu	446e476a11	go fmt	2020-09-12 04:08:03 -07:00
Chris Lu	ea26a98753	volume: validate volume correctness if last entry is a deletion	2020-09-12 04:07:04 -07:00
Chris Lu	d15682b4a1	shell: volume.balance plan by ratio of fullness	2020-09-12 04:06:26 -07:00
Chris Lu	c0ee78d2fa	adjust make file	2020-09-12 04:05:42 -07:00
Chris Lu	cd9b89ba55	reduce default wait time to 10s before shutting down	2020-09-12 04:05:33 -07:00
Chris Lu	e2c741f76f	adjust replica placement after move	2020-09-12 01:01:19 -07:00
Chris Lu	2a0925590c	filer: etcd store avoid read with nil option fix https://github.com/chrislusf/seaweedfs/issues/1463	2020-09-11 15:29:45 -07:00
Chris Lu	3984c3962f	add comment	2020-09-11 15:07:19 -07:00
Chris Lu	baa6bdf4d4	s3: listMultipartUploads fix output format	2020-09-11 15:04:01 -07:00
Chris Lu	3eda8d6dfc	s3: ListParts output xml format fix https://github.com/chrislusf/seaweedfs/issues/1461	2020-09-11 14:53:50 -07:00
Chris Lu	ab201c2798	1.97	2020-09-11 13:47:50 -07:00
Chris Lu	9d4bdfcfdf	fix volume integrity checking	2020-09-11 11:34:10 -07:00
Chris Lu	1b8094ef75	weed export: export deleted files	2020-09-11 02:05:14 -07:00
Chris Lu	12a8f5294d	test for multi dc replication	2020-09-11 01:21:17 -07:00
Chris Lu	e60b2117c3	shell: volume balance follows replica placement	2020-09-11 00:29:25 -07:00
Chris Lu	89a62e8007	refactoring	2020-09-10 23:05:00 -07:00
Chris Lu	19537c9d21	1.96	2020-09-10 19:52:07 -07:00
Chris Lu	eaf9fdde99	avoid sharing context over separate goroutine	2020-09-10 19:51:03 -07:00
Chris Lu	cc5fe6f5ee	handle special characters in html link	2020-09-10 19:46:02 -07:00
Chris Lu	401ccf1509	fix test	2020-09-10 15:13:23 -07:00
Chris Lu	c9ab8d05fa	fixes for reading deleted fid	2020-09-10 14:42:52 -07:00
ruitao.liu	5b0676049a	change elastic initialize process similar as others.	2020-09-10 23:35:20 +08:00
Chris Lu	660d7c0edd	1.95	2020-09-10 01:32:05 -07:00
Chris Lu	0d9b858cfb	Merge pull request #1455 from taozix/master add more basic elastic options.	2020-09-10 01:31:17 -07:00
ruitao.liu	9be4e97625	change logs print format.	2020-09-10 16:30:15 +08:00
ruitao.liu	719dc43af1	modify elastic urls from string to array.	2020-09-10 16:24:09 +08:00
ruitao.liu	72f9d7f047	use util to generate md5.	2020-09-10 16:11:18 +08:00
ruitao.liu	6a5b38c0d4	fix elastic kv ops.	2020-09-10 15:59:16 +08:00
ruitao.liu	3f7fbfddca	add more basic elastic options.	2020-09-10 14:22:07 +08:00
Chris Lu	b183ae54b5	master: changing 301 to 308 on redirect to volume servers fix https://github.com/chrislusf/seaweedfs/issues/1454	2020-09-09 22:36:10 -07:00
Chris Lu	47b3f932e4	watch: adjust output format	2020-09-09 22:34:48 -07:00
Chris Lu	daf0a449f7	properly cancel context for streaming grpc	2020-09-09 12:07:15 -07:00
Chris Lu	7f69acd1f2	sync pprof	2020-09-09 11:33:52 -07:00
Chris Lu	387ab6796f	filer: cross cluster synchronization	2020-09-09 11:21:23 -07:00
Chris Lu	4fc0bd1a81	return http response directly	2020-09-09 03:53:09 -07:00
Chris Lu	5e13bc878c	adjust log level	2020-09-09 03:49:26 -07:00
Chris Lu	d1b816212f	return nil if no need to do anything	2020-09-07 16:03:05 -07:00
Chris Lu	64a621bcc8	shell: volume.fix.replication also purge over replicated volumes	2020-09-07 16:00:10 -07:00
Chris Lu	c18ea21f7a	1.94	2020-09-07 13:13:53 -07:00
Chris Lu	18b98fdb72	fix test	2020-09-07 12:57:38 -07:00
Chris Lu	44b3f2efc8	filer: Elastic Search return correct kv error fix https://github.com/chrislusf/seaweedfs/issues/1452	2020-09-07 12:44:51 -07:00
Chris Lu	d80538a187	refactoring	2020-09-07 12:35:02 -07:00
Chris Lu	1a09bc43d1	refactor	2020-09-07 11:31:33 -07:00
Chris Lu	2b643f477d	typo	2020-09-06 12:47:55 -07:00
Chris Lu	05034aade5	printout over replicated locations	2020-09-06 12:44:02 -07:00
Chris Lu	432fe3d926	1.93	2020-09-06 12:12:09 -07:00
Chris Lu	d8af6f62df	adjust logging	2020-09-06 12:10:40 -07:00
Chris Lu	07f32feed0	print sync progress	2020-09-06 00:50:51 -07:00
Chris Lu	0af6252b2d	fix compilation	2020-09-06 00:50:38 -07:00
Chris Lu	ac494ff5e8	Update meta_aggregator.go	2020-09-06 00:29:16 -07:00
Chris Lu	c9f8f25ba5	read peer filer from start	2020-09-06 00:12:41 -07:00
Chris Lu	30dc365cbd	add self filer	2020-09-06 00:11:46 -07:00
Chris Lu	8acd714600	apply meta changes only if store is different	2020-09-05 23:25:03 -07:00
Chris Lu	d741ed66db	check whether shares the same filer store	2020-09-05 23:02:52 -07:00
Chris Lu	bba90ff3c8	read filer signature	2020-09-05 22:52:15 -07:00
Chris Lu	71b0e256c7	filer: setOrLoadFilerStoreSignature	2020-09-05 14:08:59 -07:00
ruitao.liu	450cf07505	skip the index that for kv usage.	2020-09-04 21:49:03 +08:00
ruitao.liu	1384ff9a2f	1.split kv in one file. 2.disable query for kv in es index.	2020-09-04 17:34:26 +08:00
ruitao.liu	83080b5e03	ES backended filer support kv ops.	2020-09-04 15:40:13 +08:00
Chris Lu	46f65a84a6	filer: elastic7 adjust default value	2020-09-03 20:12:38 -07:00
Chris Lu	f815fce81b	Merge pull request #1446 from taozix/master new filer store option to es7.	2020-09-03 20:10:23 -07:00
Chris Lu	44b4ebf0ff	filer: Redis cleanly delete directory fix https://github.com/chrislusf/seaweedfs/issues/1448	2020-09-03 19:17:39 -07:00
Chris Lu	b8f32bcab9	filer: compress stored metadata	2020-09-03 11:00:20 -07:00
Chris Lu	f76a2b2c8a	printout meta data size	2020-09-03 09:51:21 -07:00
ruitao.liu	798280e980	change filer2 to filer.	2020-09-03 17:05:26 +08:00
ruitao.liu	a93d27d1e8	new filer option to es v7.	2020-09-03 16:34:58 +08:00
Chris Lu	1d56ea24ef	fix	2020-09-03 00:08:37 -07:00
Chris Lu	7e1aad0b54	mount: map uid/gid between local and filer	2020-09-03 00:07:22 -07:00
Chris Lu	68e878adb5	fix formatting	2020-09-02 21:42:12 -07:00
Chris Lu	0d99a5da3c	c*: support kv operations	2020-09-02 18:39:24 -07:00
Chris Lu	645a4af3db	mongodb: support kv operations	2020-09-02 17:19:14 -07:00
Chris Lu	ecaa30c408	better error message	2020-09-02 17:17:44 -07:00
Chris Lu	7c770b727c	Update abstract_sql_store_kv.go	2020-09-02 10:13:48 -07:00
Chris Lu	05a6e2dc7f	sql kv upsert	2020-09-02 10:12:44 -07:00
Chris Lu	ed62f52452	fix sql insert with duplicated primary key	2020-09-02 10:09:49 -07:00
Chris Lu	9ea290aa12	fix error message	2020-09-01 23:21:41 -07:00
Chris Lu	06a1188267	mysql/postgres: support kv operations	2020-09-01 22:48:36 -07:00
Chris Lu	aa40295f06	leveldb: report not found entry	2020-09-01 22:48:23 -07:00
Chris Lu	87d7312bf6	mysql/postgres: properly report entry not found	2020-09-01 22:47:57 -07:00
Chris Lu	72b0a5f1d1	mysql or postgres: log find error	2020-09-01 22:25:17 -07:00
Chris Lu	2815bbe6c0	go fmt	2020-09-01 21:59:26 -07:00
Chris Lu	37234bf3f8	filer store adds kv support can compile now, need to implement those unimplemented	2020-09-01 21:59:00 -07:00
James Hartig	8e54e34576	volume: Don't unmount before deleting volume in copy If we unmount first and then delete, the delete fails because the volume was unmounted. Delete ends up doing the same thing as the unmount anyways.	2020-09-01 22:00:07 -04:00
Chris Lu	2b14ae5819	fix tests	2020-09-01 01:33:43 -07:00
Chris Lu	d91ec535b3	fix tests	2020-09-01 01:29:13 -07:00
Chris Lu	eb7929a971	rename filer2 to filer	2020-09-01 00:21:19 -07:00
Chris Lu	38e06d783d	volume: check disk space before compaction fix https://github.com/chrislusf/seaweedfs/issues/1440	2020-08-31 18:10:53 -07:00
Chris Lu	dbb10e0f0b	testing with more than 30days	2020-08-31 17:59:29 -07:00
Chris Lu	dd176cdb8b	avoid tiny step pagination	2020-08-31 11:28:03 -07:00
Chris Lu	97733c3dff	typo	2020-08-31 10:49:17 -07:00
Chris Lu	c8398bdb09	adjust metrics	2020-08-31 10:41:05 -07:00
Chris Lu	8a1c8e41b3	simplify if else logic	2020-08-31 10:39:24 -07:00
Chris Lu	f0c89cfacd	go fmt	2020-08-31 10:23:31 -07:00
Chris Lu	edb9d65e05	Merge pull request #1431 from kmlebedev/wip-prefix-search Wip prefix search	2020-08-31 10:22:14 -07:00
Konstantin Lebedev	9a195bebfd	accurate limit	2020-08-31 22:13:13 +05:00
Konstantin Lebedev	18b98f9747	not convert err to string	2020-08-31 21:55:18 +05:00
Konstantin Lebedev	22f32e75c4	rename	2020-08-31 21:52:05 +05:00
Konstantin Lebedev	a8de332beb	fix	2020-08-31 21:45:44 +05:00
Konstantin Lebedev	7f013d77d6	add var UnsupportedListDirectoryPrefixedErr	2020-08-31 21:43:32 +05:00
Konstantin Lebedev	60a86cfe04	add const	2020-08-31 21:35:16 +05:00
Konstantin Lebedev	82ea121d09	rm func	2020-08-31 18:50:06 +05:00
Konstantin Lebedev	2ea638f865	filer store wrapper can implement the logic to filter by prefi	2020-08-31 17:13:56 +05:00
Chris Lu	408e339c53	also delete the manifest chunk itself	2020-08-31 00:16:03 -07:00
Chris Lu	dd52037c27	1.92	2020-08-30 22:56:35 -07:00
Chris Lu	33d8c6c617	change log to warning	2020-08-30 21:01:44 -07:00
Chris Lu	e62d7f221f	a bit more logs	2020-08-30 21:00:24 -07:00
Chris Lu	22fe4ae573	mount: fix for UrBackup	2020-08-30 21:00:09 -07:00
Chris Lu	ae84a9ee2e	add logs	2020-08-30 20:59:04 -07:00
Chris Lu	a41588279a	change log level 5 to 4	2020-08-30 20:12:04 -07:00
Chris Lu	56244fb9a1	fix hard coded host address	2020-08-30 10:23:35 -07:00
Chris Lu	f2a8574448	filer and mount deletion resolves manifest chunks also	2020-08-30 02:07:14 -07:00
Chris Lu	99ecf63276	go fmt	2020-08-29 22:28:33 -07:00
Chris Lu	ef75ce8a34	use UTC for all time related folders	2020-08-29 21:01:14 -07:00
Chris Lu	b69cb74c03	read meta logs by timestamp pass in event ts when moving logs meta aggregator reads in memory logs only	2020-08-29 17:37:19 -07:00
Chris Lu	063c9ddac5	adjust logs	2020-08-29 11:56:22 -07:00
Chris Lu	ca658a97c5	add signatures to messages to avoid double processing	2020-08-28 23:48:48 -07:00
Chris Lu	63ad1abcce	watch: follow changes for a pattern	2020-08-28 19:43:04 -07:00
Chris Lu	b7a654a318	file mime avoid saving application/octet-stream	2020-08-28 19:42:40 -07:00
Chris Lu	707192f966	fix help message	2020-08-27 00:13:38 -07:00
Chris Lu	0be6863c87	rename	2020-08-26 22:40:15 -07:00
Chris Lu	50f7e2e89f	add log messages	2020-08-26 09:51:20 -07:00
Chris Lu	6bc618051c	log message	2020-08-26 09:32:56 -07:00
Chris Lu	9dae4e9d3e	fix logs	2020-08-26 09:31:11 -07:00
Chris Lu	4c1a3187bf	fix log error	2020-08-26 09:25:54 -07:00
Chris Lu	ab759f0ec2	erasure coding: fix EC error if multiple disks are configured in one volume server	2020-08-26 09:16:58 -07:00
Chris Lu	d1cf39f180	fix logging	2020-08-26 08:55:15 -07:00
Chris Lu	83012f543a	s: avoid possible completeMultipartUpload re-tries that can overwrite the uploaded file	2020-08-26 08:50:24 -07:00
Konstantin Lebedev	464d4c82ec	stop send heartbeat before stop volume server	2020-08-26 16:16:11 +05:00
Chris Lu	1901f15cd2	volume server: remove whitelist for status checking Volume liveness/readiness not work if jwt enabled, I try use param -whiteList but jwt breaks	2020-08-25 02:27:47 -07:00
Chris Lu	81e5124faf	s3: list bucket permission change from admin to read fix https://github.com/chrislusf/seaweedfs/issues/1430	2020-08-24 11:22:45 -07:00
Chris Lu	5f55a87101	close http response	2020-08-24 00:32:44 -07:00
Chris Lu	0b301b0b47	consume all response body in order to release requests	2020-08-23 21:32:29 -07:00
Chris Lu	2cdad77d82	fix checkin error	2020-08-23 18:30:11 -07:00
Chris Lu	81a8aa6581	1.91	2020-08-23 17:28:40 -07:00
Chris Lu	51346a5930	always keep the manifest list of chunks	2020-08-23 17:15:12 -07:00
Chris Lu	c1d1677a28	keep manifest chunks forever	2020-08-23 16:59:01 -07:00
Chris Lu	98175548c2	adjust deletion ordering	2020-08-23 16:27:12 -07:00
Chris Lu	f0e50e7b26	fix test	2020-08-23 16:12:34 -07:00
Chris Lu	aee27ccbe1	multiple fixes * adjust isOpen count * move ContinuousDirtyPages lock to filehandle * fix problem with MergeIntoVisibles, avoid reusing slices * let filer delete the garbage	2020-08-23 15:48:02 -07:00
Chris Lu	c19245886c	simpler test	2020-08-23 14:20:27 -07:00
Chris Lu	df816a58fe	add tests	2020-08-23 14:09:25 -07:00
Chris Lu	77393d3d30	add file handle locking when changing file entry	2020-08-23 00:35:50 -07:00
Chris Lu	d60bcbf08a	sorting chunks	2020-08-23 00:00:36 -07:00
Chris Lu	5e6b714836	add random test	2020-08-22 16:33:00 -07:00
Chris Lu	42ddbcc3a2	SQL insert falls back to update fix https://github.com/chrislusf/seaweedfs/issues/1429	2020-08-20 07:52:46 -07:00
Chris Lu	f48567c5c6	remove unused function	2020-08-19 22:53:49 -07:00
Chris Lu	c45ba5d7d4	fix listObjectsV2 response format fix https://github.com/chrislusf/seaweedfs/issues/1426 issue 1	2020-08-19 12:08:04 -07:00
James Hartig	3ccfa4c6ad	Added VolumeMarkWritable and VolumeStatus grpc methods This is necessary for copy to mark as read-only and then restore the original state afterwards.	2020-08-19 11:42:56 -04:00
Chris Lu	3b4b1d4a77	fix tests	2020-08-19 01:37:56 -07:00
Chris Lu	839634097f	also do flush on release	2020-08-19 01:27:10 -07:00
Chris Lu	c27e18aa6a	read possible old deleted chunks	2020-08-19 00:43:07 -07:00
Chris Lu	a78772d5ea	avoid shutdown in the middle of running	2020-08-19 00:42:02 -07:00
Chris Lu	ae9bc4a508	logs	2020-08-18 23:42:26 -07:00
Chris Lu	ed4b43b419	adjust logs	2020-08-18 23:42:09 -07:00
Chris Lu	9f1e0aeef5	delete chunks in the last step	2020-08-18 23:40:53 -07:00
Chris Lu	99d05f758c	adjust logs	2020-08-18 23:39:18 -07:00
Chris Lu	fe01191b5b	support read option readDeleted=true	2020-08-18 19:22:16 -07:00
Chris Lu	6ccd7f0a4d	refactoring	2020-08-18 18:01:37 -07:00
Chris Lu	c026eb0592	refactoring	2020-08-18 17:39:29 -07:00
Chris Lu	7e91ae592c	pass in option to read deleted entries not working yet	2020-08-18 17:37:26 -07:00
Chris Lu	51ecb49db3	for debugging	2020-08-18 17:35:39 -07:00
Chris Lu	ee11d98650	refactoring	2020-08-18 17:35:19 -07:00
Chris Lu	6a92f0bc7a	refactoring to typed Size Go is amazing with refactoring!	2020-08-18 17:04:28 -07:00
Chris Lu	618b2f6829	release resources only when needed to	2020-08-18 12:53:08 -07:00
Chris Lu	208849702d	logs	2020-08-18 12:52:54 -07:00
Chris Lu	1fcd083db3	printout data size	2020-08-18 09:09:29 -07:00
Chris Lu	3e5339337a	minor	2020-08-18 08:50:14 -07:00
Chris Lu	85001cbec7	properly report io.EOF	2020-08-18 08:18:54 -07:00
Chris Lu	0ca45a5cbc	adjust logs	2020-08-18 00:34:15 -07:00
Chris Lu	cd43738245	fix reading when filling zeros	2020-08-18 00:32:01 -07:00
Chris Lu	ecb3ce46be	adjust error logs	2020-08-18 00:31:42 -07:00
Chris Lu	30fe424469	add one more test case	2020-08-17 22:47:27 -07:00
Chris Lu	1b68ba953b	fix for out of range reads	2020-08-17 22:46:32 -07:00
Chris Lu	56fbd2c211	fix reading	2020-08-17 21:17:32 -07:00
Chris Lu	09e126bae5	refactoring: use interface	2020-08-17 20:20:08 -07:00
Chris Lu	be4d42b8e2	rename	2020-08-17 20:15:53 -07:00
Chris Lu	0625e63648	count 0 as part of the reads	2020-08-17 20:14:40 -07:00
Chris Lu	0be4b6e7f0	logs	2020-08-17 16:05:40 -07:00
Chris Lu	97e54a80d4	rename variables	2020-08-17 16:05:13 -07:00
Chris Lu	abdaf9958d	possibly read more	2020-08-17 16:04:56 -07:00
Chris Lu	9d46c7bc78	rename	2020-08-17 11:12:10 -07:00
Chris Lu	4ccfdaeb4d	prevent nil	2020-08-17 10:07:34 -07:00
Chris Lu	24c8e6bcb4	minor optimization	2020-08-17 10:03:34 -07:00
Chris Lu	f5837b7000	report error first	2020-08-16 23:49:10 -07:00
Chris Lu	2ac27616bc	fix possible out of range bytes avoid buff out of range resp.Data = buff[:totalRead]	2020-08-16 23:47:34 -07:00
Chris Lu	ff200398bb	1.90	2020-08-16 21:18:50 -07:00
Chris Lu	4ceeba9e70	streaming reads	2020-08-16 21:07:46 -07:00
Chris Lu	e72953dff7	logs	2020-08-16 21:06:03 -07:00
Chris Lu	627b081b67	adjust logs	2020-08-16 16:32:22 -07:00
Chris Lu	22e5132b3a	adjust log level	2020-08-16 16:25:11 -07:00
Chris Lu	ee0f92a6be	reduce memory allocation	2020-08-16 16:24:40 -07:00
Chris Lu	2ba817afac	read randomly written data	2020-08-16 15:16:46 -07:00
Chris Lu	6111b265e7	fix compilation	2020-08-16 01:38:16 -07:00
Chris Lu	4a77f0820a	clean up logs	2020-08-16 01:37:50 -07:00
Chris Lu	8c9e6eaacd	fix tests	2020-08-16 01:35:52 -07:00
Chris Lu	b71df82292	1.89	2020-08-16 00:57:28 -07:00
Chris Lu	1d9ea30b72	fix ViewFromVisibleIntervals	2020-08-16 00:49:26 -07:00
Chris Lu	aec7f32b02	fix reader_at	2020-08-16 00:49:08 -07:00
Chris Lu	5d80fc2ec7	adjust logs	2020-08-15 21:09:31 -07:00
Chris Lu	e0bfd3161a	update metadata only if changed	2020-08-15 19:56:24 -07:00
Chris Lu	003d48da21	adjust logs	2020-08-15 19:55:28 -07:00
Chris Lu	6ee8d952d2	adjust log level	2020-08-15 18:24:35 -07:00
Chris Lu	3e1395b767	adjust log message	2020-08-15 17:06:16 -07:00
Chris Lu	a22ee30596	fix nil	2020-08-15 17:01:42 -07:00
Chris Lu	0d60e67816	ensure meta data changes are updated	2020-08-15 14:15:07 -07:00
Chris Lu	5b43bddf20	proper deletion ordering delete central file store first, then delete local cache	2020-08-15 09:33:41 -07:00
Chris Lu	c647deace1	file size support set file length use Attr.FileSize and TotalChunkSize to determine file size	2020-08-15 09:32:47 -07:00
Chris Lu	c03bb180eb	fix error reporting	2020-08-14 00:44:02 -07:00
Chris Lu	eb493283dd	deletion error report	2020-08-14 00:23:01 -07:00
Chris Lu	a7f669044e	rename also applies to open file handle	2020-08-14 00:22:49 -07:00
Chris Lu	edfa73782f	adjust log level	2020-08-14 00:22:21 -07:00
Chris Lu	0983060a90	increase default volume file size limit to 1024 avoid possible large chunk size set on mount or filer	2020-08-13 09:07:22 -07:00
Chris Lu	090612492c	fix compilation	2020-08-13 00:07:56 -07:00
Chris Lu	d43129d27f	add back handles lock	2020-08-12 23:52:13 -07:00
Chris Lu	f735d579d3	adjust comment	2020-08-12 13:22:19 -07:00
Chris Lu	8824a9755c	remove directory cache	2020-08-12 13:11:04 -07:00
Chris Lu	83cad3da79	add retry file upload	2020-08-11 20:30:11 -07:00
Chris Lu	152a6cbc2b	minor adjustments	2020-08-10 20:42:27 -07:00
Chris Lu	f1b40567e3	Merge pull request #1419 from robin1900/master fix bug: two same volumeId in different collections	2020-08-10 20:38:55 -07:00
cheng.li01	25fbff5d52	fix bug: two same volumeId in different collections 1, there will be two leader when master server startup in a few seconds 2, raft server will get a leader even there is only one master, so there is no need to do hard code to set the server to be leader	2020-08-10 16:37:47 +08:00
Chris Lu	e74dc4e4bc	add back fs node cache for renaming	2020-08-09 21:56:09 -07:00
Chris Lu	f86c7d911a	remove println	2020-08-09 14:42:25 -07:00
Chris Lu	9ecc1170a3	istObjects和listObjectsV2不能查询子目录 #1418 fix https://github.com/chrislusf/seaweedfs/issues/1418	2020-08-09 14:35:53 -07:00
Chris Lu	3f4aff5dde	s3: fix delimiter in list response	2020-08-09 09:09:35 -07:00
Chris Lu	b056707770	1.88	2020-08-08 22:55:12 -07:00
Chris Lu	9832653e1d	FUSE mount: proper error with deleting non empty folder	2020-08-08 21:37:36 -07:00
Chris Lu	3b1a95ac26	filer refactoring: same auto chunking logic for POST and PUT, no size limit	2020-08-08 12:02:06 -07:00
Chris Lu	67348e7b15	less noisy heartbeat logs	2020-08-08 10:53:35 -07:00
Chris Lu	bee0d7e5eb	lower log priority for noisy heartbeat	2020-08-08 10:52:13 -07:00
Chris Lu	ae00cce4bd	support POST and PUT auto chunking	2020-08-08 10:45:37 -07:00
Chris Lu	bd8bfdae07	refactoring	2020-08-08 10:18:43 -07:00
Chris Lu	ab6e5c0dc4	adjust error message	2020-08-08 10:18:32 -07:00
Chris Lu	828a5ae429	check signature only when auth is enabled	2020-08-08 09:11:40 -07:00
Chris Lu	20e2ac1add	filer: store md5 metadata for files uploaded by filer fix https://github.com/chrislusf/seaweedfs/issues/1412	2020-08-06 10:04:17 -07:00
Chris Lu	93ea0801ea	volume: the variable for the master node may be stale? related to https://github.com/chrislusf/seaweedfs/issues/1414	2020-08-06 09:48:54 -07:00
Chris Lu	4ecfa9879d	volume: report Content-MD5 in response header	2020-08-06 05:22:53 -07:00
Chris Lu	41007ced77	remove logging	2020-08-06 04:26:29 -07:00
Chris Lu	2b74abf766	S3: configurable access for anonymous user fix https://github.com/chrislusf/seaweedfs/issues/1413	2020-08-06 03:41:34 -07:00
Устюжанин Антон Александрович	9ca011e3ca	ListDirectoryPrefixedEntries	2020-08-06 00:38:59 +05:00
Устюжанин Антон Александрович	b231f7bdab	ListDirectoryPrefixedEntries	2020-08-06 00:37:42 +05:00
Устюжанин Антон Александрович	a457c308ad	test ListDirectoryPrefixedEntries	2020-08-06 00:24:31 +05:00
Устюжанин Антон Александрович	2d3b355fb6	test ListDirectoryPrefixedEntries	2020-08-05 23:56:06 +05:00
Устюжанин Антон Александрович	67f134ebd7	test ListDirectoryPrefixedEntries	2020-08-05 23:44:48 +05:00
Устюжанин Антон Александрович	dc9fc01254	test ListDirectoryPrefixedEntries	2020-08-05 23:38:00 +05:00
Устюжанин Антон Александрович	33a9e5e2d1	test ListDirectoryPrefixedEntries	2020-08-05 22:19:16 +05:00
Chris Lu	cbd80253e3	better needle id format	2020-08-04 22:09:07 -07:00
Chris Lu	1b3a80dd3d	non-fatal error	2020-08-01 13:46:52 -07:00
Chris Lu	bbbea8159c	http request use gzip if possible	2020-08-01 13:20:52 -07:00

... 48 49 50 51 52 ...

7222 Commits