mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-12-25 02:07:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

keep update from original (#1)

Browse Source 
keep update from original
This commit is contained in:
bingoohuang 2019-07-16 11:13:23 +08:00 committed by GitHub
commit d19bbee98d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
293 changed files with 19032 additions and 4932 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.github/ISSUE_TEMPLATE/bug_report.md vendored
View File

@ -1,11 +1,19 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: ''
assignees: ''
---
Sponsors SeaweedFS via Patreon https://www.patreon.com/seaweedfs
example of a good issue report:
https://github.com/chrislusf/seaweedfs/issues/1005
example of a bad issue report:
https://github.com/chrislusf/seaweedfs/issues/1008
**Describe the bug**
A clear and concise description of what the bug is.

3
.gitignore vendored
View File

@ -80,3 +80,6 @@ build
target
*.class
other/java/hdfs/dependency-reduced-pom.xml
# binary file
weed/weed

12
.travis.yml
View File

@ -3,13 +3,16 @@ language: go
go:
- 1.10.x
- 1.11.x
- tip
- 1.12.x
# - tip
before_install:
- export PATH=/home/travis/gopath/bin:$PATH
install:
- go get ./weed/...
- export CGO_ENABLED="0"
- go env
- go get -u ./weed/...
script:
- go test ./weed/...
@ -26,9 +29,12 @@ deploy:
- build/linux_arm64.tar.gz
- build/linux_386.tar.gz
- build/linux_amd64.tar.gz
- build/linux_amd64_large_disk.tar.gz
- build/darwin_amd64.tar.gz
- build/darwin_amd64_large_disk.tar.gz
- build/windows_386.zip
- build/windows_amd64.zip
- build/windows_amd64_large_disk.zip
- build/freebsd_arm.tar.gz
- build/freebsd_amd64.tar.gz
- build/freebsd_386.tar.gz
@ -41,4 +47,4 @@ deploy:
on:
tags: true
repo: chrislusf/seaweedfs
go: tip
go: 1.12.x

17
Makefile
View File

@ -12,6 +12,9 @@ build = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -ldflags "-extldflags -stat
tar = cd build && tar -cvzf $(1)_$(2).tar.gz $(appname)$(3) && rm $(appname)$(3)
zip = cd build && zip $(1)_$(2).zip $(appname)$(3) && rm $(appname)$(3)
build_large = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -tags 5BytesOffset -ldflags "-extldflags -static" -o build/$(appname)$(3) $(SOURCE_DIR)
tar_large = cd build && tar -cvzf $(1)_$(2)_large_disk.tar.gz $(appname)$(3) && rm $(appname)$(3)
zip_large = cd build && zip $(1)_$(2)_large_disk.zip $(appname)$(3) && rm $(appname)$(3)
all: build
@ -32,9 +35,21 @@ linux: deps
mkdir -p linux
GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -o linux/$(BINARY) $(SOURCE_DIR)
release: deps windows_build darwin_build linux_build bsd_build
release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_darwin_build 5_byte_windows_build
##### LINUX BUILDS #####
5_byte_linux_build:
$(call build_large,linux,amd64,)
$(call tar_large,linux,amd64)
5_byte_darwin_build:
$(call build_large,darwin,amd64,)
$(call tar_large,darwin,amd64)
5_byte_windows_build:
$(call build_large,windows,amd64,.exe)
$(call zip_large,windows,amd64,.exe)
linux_build: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz
build/linux_386.tar.gz: $(sources)

120
README.md
View File

@ -15,13 +15,11 @@ possible entirely thanks to the support of these awesome [backers](https://githu
If you'd like to grow SeaweedFS even stronger, please consider joining our
<a href="https://www.patreon.com/seaweedfs">sponsors on Patreon</a>.
Platinum ($2500/month), Gold ($500/month): put your company logo on the SeaweedFS github page
Generous Backer($50/month), Backer($10/month): put your name on the SeaweedFS backer page.
Your support will be really appreciated by me and other supporters!
<h3 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a></h3>
<!--
<h4 align="center">Platinum</h4>
<p align="center">
@ -45,6 +43,8 @@ Your support will be really appreciated by me and other supporters!
</tbody>
</table>
-->
---
@ -52,9 +52,29 @@ Your support will be really appreciated by me and other supporters!
- [SeaweedFS on Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTc4MmVlYmFlNjBmZTgzZmJlYmI1MDE1YzkyNWYyZjkwZDFiM2RlMDdjNjVlNjdjYzc4NGFhZGIyYzEyMzJkYTA)
- [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
- [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
- [SeaweedFS Introduction Slides](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
Table of Contents
=================
## Introduction
* [Introduction](#introduction)
* [Features](#features)
* [Additional Features](#additional-features)
* [Filer Features](#filer-features)
* [Example Usage](#example-usage)
* [Architecture](#architecture)
* [Compared to Other File Systems](#compared-to-other-file-systems)
* [Compared to HDFS](#compared-to-hdfs)
* [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph)
* [Compared to GlusterFS](#compared-to-glusterfs)
* [Compared to Ceph](#compared-to-ceph)
* [Dev Plan](#dev-plan)
* [Installation Guide](#installation-guide)
* [Disk Related Topics](#disk-related-topics)
* [Benchmark](#Benchmark)
* [License](#license)
## Introduction ##
SeaweedFS is a simple and highly scalable distributed file system. There are two objectives:
@ -65,41 +85,57 @@ SeaweedFS started as an Object Store to handle small files efficiently. Instead
There is only 40 bytes of disk storage overhead for each file's metadata. It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases.
SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf).
SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). Also, SeaweedFS implements erasure coding with ideas from [f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf)
SeaweedFS can work very well with just the object store. [[Filer]] can then be added later to support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql/Postgres/Redis/Cassandra/LevelDB.
## Additional Features
* Can choose no replication or different replication levels, rack and data center aware
* Automatic master servers failover - no single point of failure (SPOF)
* Automatic Gzip compression depending on file mime type
* Automatic compaction to reclaim disk space after deletion or update
* Servers in the same cluster can have different disk spaces, file systems, OS etc.
* Adding/Removing servers does **not** cause any data re-balancing
* Optionally fix the orientation for jpeg pictures
* Support Etag, Accept-Range, Last-Modified, etc.
* Support in-memory/leveldb/boltdb/btree mode tuning for memory/performance balance.
[Back to TOC](#table-of-contents)
## Filer Features
## Features ##
[Back to TOC](#table-of-contents)
## Additional Features ##
* Can choose no replication or different replication levels, rack and data center aware.
* Automatic master servers failover - no single point of failure (SPOF).
* Automatic Gzip compression depending on file mime type.
* Automatic compaction to reclaim disk space after deletion or update.
* Servers in the same cluster can have different disk spaces, file systems, OS etc.
* Adding/Removing servers does **not** cause any data re-balancing.
* Optionally fix the orientation for jpeg pictures.
* Support ETag, Accept-Range, Last-Modified, etc.
* Support in-memory/leveldb/boltdb/btree mode tuning for memory/performance balance.
* Support rebalancing the writable and readonly volumes.
[Back to TOC](#table-of-contents)
## Filer Features ##
* [filer server][Filer] provide "normal" directories and files via http.
* [mount filer][Mount] to read and write files directly as a local directory via FUSE.
* [Amazon S3 compatible API][AmazonS3API] to access files with S3 tooling.
* [Erasure Coding for warm storage][ErasureCoding] Rack-Aware 10.4 erasure coding reduces storage cost and increases availability.
* [Hadoop Compatible File System][Hadoop] to access files from Hadoop/Spark/Flink/etc jobs.
* [Async Backup To Cloud][BackupToCloud] has extremely fast local access and backups to Amazon S3, Google Cloud Storage, Azure, BackBlaze.
* [WebDAV] access as a mapped drive on Mac and Windows, or from mobile devices.
[Filer]: https://github.com/chrislusf/seaweedfs/wiki/Directories-and-Files
[Mount]: https://github.com/chrislusf/seaweedfs/wiki/Mount
[AmazonS3API]: https://github.com/chrislusf/seaweedfs/wiki/Amazon-S3-API
[BackupToCloud]: https://github.com/chrislusf/seaweedfs/wiki/Backup-to-Cloud
[Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System
[WebDAV]: https://github.com/chrislusf/seaweedfs/wiki/WebDAV
[ErasureCoding]: https://github.com/chrislusf/seaweedfs/wiki/Erasure-coding-for-warm-storage
[Back to TOC](#table-of-contents)
## Example Usage ##
## Example Usage
By default, the master node runs on port 9333, and the volume nodes run on port 8080.
Let's start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. We'll use localhost as an example.
SeaweedFS uses HTTP REST operations to read, write, and delete. The responses are in JSON or JSONP format.
### Start Master Server
### Start Master Server ###
```
> ./weed master
@ -125,7 +161,7 @@ Second, to store the file content, send a HTTP multi-part POST request to `url +
```
> curl -F file=@/home/chris/myphoto.jpg http://127.0.0.1:8080/3,01637037d6
{"size": 43234}
{"name":"myphoto.jpg","size":43234,"eTag":"1cc0118e"}
```
To update, send another POST request with updated file content.
@ -135,6 +171,7 @@ For deletion, send an HTTP DELETE request to the same `url + '/' + fid` URL:
```
> curl -X DELETE http://127.0.0.1:8080/3,01637037d6
```
### Save File Id ###
Now, you can save the `fid`, 3,01637037d6 in this case, to a database field.
@ -157,7 +194,7 @@ First look up the volume server's URLs by the file's volumeId:
```
> curl http://localhost:9333/dir/lookup?volumeId=3
{"locations":[{"publicUrl":"localhost:8080","url":"localhost:8080"}]}
{"volumeId":"3","locations":[{"publicUrl":"localhost:8080","url":"localhost:8080"}]}
```
Since (usually) there are not too many volume servers, and volumes don't move often, you can cache the results most of the time. Depending on the replication type, one volume can have multiple replica locations. Just randomly pick one location to read.
@ -213,7 +250,7 @@ More details about replication can be found [on the wiki][Replication].
You can also set the default replication strategy when starting the master server.
### Allocate File Key on specific data center ###
### Allocate File Key on Specific Data Center ###
Volume servers can be started with a specific data center name:
@ -239,6 +276,8 @@ When requesting a file key, an optional "dataCenter" parameter can limit the ass
[feat-3]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#upload-large-files
[feat-4]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#collection-as-a-simple-name-space
[Back to TOC](#table-of-contents)
## Architecture ##
Usually distributed file systems split each file into chunks, a central master keeps a mapping of filenames, chunk indices to chunk handles, and also which chunks each chunk server has.
@ -279,12 +318,16 @@ Each individual file size is limited to the volume size.
All file meta information stored on an volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
[Back to TOC](#table-of-contents)
## Compared to Other File Systems ##
Most other distributed file systems seem more complicated than necessary.
SeaweedFS is meant to be fast and simple, in both setup and operation. If you do not understand how it works when you reach here, we've failed! Please raise an issue with any questions or update this file with clarifications.
[Back to TOC](#table-of-contents)
### Compared to HDFS ###
HDFS uses the chunk approach for each file, and is ideal for storing large files.
@ -293,6 +336,7 @@ SeaweedFS is ideal for serving relatively smaller files quickly and concurrently
SeaweedFS can also store extra large files by splitting them into manageable data chunks, and store the file ids of the data chunks into a meta chunk. This is managed by "weed upload/download" tool, and the weed master or volume servers are agnostic about it.
[Back to TOC](#table-of-contents)
### Compared to GlusterFS, Ceph ###
@ -310,12 +354,16 @@ The architectures are mostly the same. SeaweedFS aims to store and read files fa
| GlusterFS | hashing | | FUSE, NFS | | |
| Ceph | hashing + rules | | FUSE | Yes | |
[Back to TOC](#table-of-contents)
### Compared to GlusterFS ###
GlusterFS stores files, both directories and content, in configurable volumes called "bricks".
GlusterFS hashes the path and filename into ids, and assigned to virtual volumes, and then mapped to "bricks".
[Back to TOC](#table-of-contents)
### Compared to Ceph ###
Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/chrislusf/seaweedfs/issues/120)
@ -336,16 +384,26 @@ SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Redis, Cassa
| Volume | OSD | optimized for small files |
| Filer | Ceph FS | linearly scalable, Customizable, O(1) or O(logN) |
[Back to TOC](#table-of-contents)
## Dev plan ##
## Dev Plan ##
More tools and documentation, on how to maintain and scale the system. For example, how to move volumes, automatically balancing data, how to grow volumes, how to check system status, etc.
Other key features include: Erasure Encoding, JWT security.
This is a super exciting project! And we need helpers and [support](https://www.patreon.com/seaweedfs)!
BTW, We suggest run the code style check script `util/gostd` before you push your branch to remote, it will make SeaweedFS easy to review, maintain and develop:
## Installation guide for users who are not familiar with golang
```
$ ./util/gostd
```
[Back to TOC](#table-of-contents)
## Installation Guide ##
> Installation guide for users who are not familiar with golang
Step 1: install go on your machine and setup the environment by following the instructions at:
@ -372,17 +430,21 @@ Step 4: after you modify your code locally, you could start a local build by cal
$GOPATH/src/github.com/chrislusf/seaweedfs/weed
```
## Disk Related topics ##
[Back to TOC](#table-of-contents)
## Disk Related Topics ##
### Hard Drive Performance ###
When testing read performance on SeaweedFS, it basically becomes a performance test of your hard drive's random read speed. Hard drives usually get 100MB/s~200MB/s.
### Solid State Disk
### Solid State Disk ###
To modify or delete small files, SSD must delete a whole block at a time, and move content in existing blocks to a new block. SSD is fast when brand new, but will get fragmented over time and you have to garbage collect, compacting blocks. SeaweedFS is friendly to SSD since it is append-only. Deletion and compaction are done on volume level in the background, not slowing reading and not causing fragmentation.
## Benchmark
[Back to TOC](#table-of-contents)
## Benchmark ##
My Own Unscientific Single Machine Results on Mac Book with Solid State Disk, CPU: 1 Intel Core i7 2.6GHz.
@ -435,8 +497,9 @@ Percentage of the requests served within a certain time (ms)
100% 20.7 ms
```
[Back to TOC](#table-of-contents)
## License
## License ##
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -450,7 +513,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
[Back to TOC](#table-of-contents)
## Stargazers over time
## Stargazers over time ##
[![Stargazers over time](https://starcharts.herokuapp.com/chrislusf/seaweedfs.svg)](https://starcharts.herokuapp.com/chrislusf/seaweedfs)

13
docker/Dockerfile
View File

@ -1,9 +1,20 @@
FROM frolvlad/alpine-glibc
# Supercronic install settings
ENV SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.8/supercronic-linux-amd64 \
SUPERCRONIC=supercronic-linux-amd64 \
SUPERCRONIC_SHA1SUM=be43e64c45acd6ec4fce5831e03759c89676a0ea
# Install SeaweedFS and Supercronic ( for cron job mode )
# Tried to use curl only (curl -o /tmp/linux_amd64.tar.gz ...), however it turned out that the following tar command failed with "gzip: stdin: not in gzip format"
RUN apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \
wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/latest | egrep -o 'chrislusf/seaweedfs/releases/download/.*/linux_amd64.tar.gz') && \
tar -C /usr/bin/ -xzvf /tmp/linux_amd64.tar.gz && \
curl -fsSLO "$SUPERCRONIC_URL" && \
echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - && \
chmod +x "$SUPERCRONIC" && \
mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \
apk del build-dependencies && \
rm -rf /tmp/*
@ -22,6 +33,8 @@ EXPOSE 9333
# s3 server http port
EXPOSE 8333
RUN mkdir -p /data/filerldb2
VOLUME /data
COPY filer.toml /etc/seaweedfs/filer.toml

2
docker/Dockerfile.go_build
View File

@ -16,6 +16,8 @@ EXPOSE 9333
# s3 server http port
EXPOSE 8333
RUN mkdir -p /data/filerldb2
VOLUME /data
RUN mkdir -p /etc/seaweedfs

13
docker/entrypoint.sh
View File

@ -29,11 +29,10 @@ case "$1" in
;;
'filer')
ARGS="-ip `hostname -i` "
ARGS=""
if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
ARGS="$ARGS -master=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
fi
mkdir -p /data/filerdb
exec /usr/bin/weed $@ $ARGS
;;
@ -45,6 +44,16 @@ case "$1" in
exec /usr/bin/weed $@ $ARGS
;;
'cronjob')
MASTER=${WEED_MASTER-localhost:9333}
FIX_REPLICATION_CRON_SCHEDULE=${CRON_SCHEDULE-*/7 * * * * *}
echo "$FIX_REPLICATION_CRON_SCHEDULE" 'echo "volume.fix.replication" | weed shell -master='$MASTER > /crontab
BALANCING_CRON_SCHEDULE=${CRON_SCHEDULE-25 * * * * *}
echo "$BALANCING_CRON_SCHEDULE" 'echo "volume.balance -c ALL -force" | weed shell -master='$MASTER >> /crontab
echo "Running Crontab:"
cat /crontab
exec supercronic /crontab
;;
*)
exec /usr/bin/weed $@
;;

4
docker/filer.toml
View File

@ -1,3 +1,3 @@
[leveldb]
[leveldb2]
enabled = true
dir = "/data/filerdb"
dir = "/data/filerldb2"

10
docker/seaweedfs-compose.yml
View File

@ -26,6 +26,16 @@ services:
depends_on:
- master
- volume
cronjob:
image: chrislusf/seaweedfs # use a remote image
command: 'cronjob'
environment:
# Run re-replication every 2 minutes
CRON_SCHEDULE: '*/2 * * * * *' # Default: '*/5 * * * * *'
WEED_MASTER: master:9333 # Default: localhost:9333
depends_on:
- master
- volume
s3:
image: chrislusf/seaweedfs # use a remote image
ports:

2
other/java/client/pom.xml
View File

@ -4,7 +4,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>1.0.5</version>
<version>1.1.0</version>
<parent>
<groupId>org.sonatype.oss</groupId>

59
other/java/client/src/main/java/seaweedfs/client/FilerClient.java
View File

@ -57,6 +57,20 @@ public class FilerClient {
}
public boolean mv(String oldPath, String newPath) {
Path oldPathObject = Paths.get(oldPath);
String oldParent = oldPathObject.getParent().toString();
String oldName = oldPathObject.getFileName().toString();
Path newPathObject = Paths.get(newPath);
String newParent = newPathObject.getParent().toString();
String newName = newPathObject.getFileName().toString();
return atomicRenameEntry(oldParent, oldName, newParent, newName);
}
public boolean rm(String path, boolean isRecursive) {
Path pathObject = Paths.get(path);
@ -159,21 +173,27 @@ public class FilerClient {
}
public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit) {
return filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
List<FilerProto.Entry> entries = filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
.setDirectory(path)
.setPrefix(entryPrefix)
.setStartFromFileName(lastEntryName)
.setLimit(limit)
.build()).getEntriesList();
List<FilerProto.Entry> fixedEntries = new ArrayList<>(entries.size());
for (FilerProto.Entry entry : entries) {
fixedEntries.add(fixEntryAfterReading(entry));
}
return fixedEntries;
}
public FilerProto.Entry lookupEntry(String directory, String entryName) {
try {
return filerGrpcClient.getBlockingStub().lookupDirectoryEntry(
FilerProto.Entry entry = filerGrpcClient.getBlockingStub().lookupDirectoryEntry(
FilerProto.LookupDirectoryEntryRequest.newBuilder()
.setDirectory(directory)
.setName(entryName)
.build()).getEntry();
return fixEntryAfterReading(entry);
} catch (Exception e) {
LOG.warn("lookupEntry {}/{}: {}", directory, entryName, e);
return null;
@ -222,4 +242,39 @@ public class FilerClient {
return true;
}
public boolean atomicRenameEntry(String oldParent, String oldName, String newParent, String newName) {
try {
filerGrpcClient.getBlockingStub().atomicRenameEntry(FilerProto.AtomicRenameEntryRequest.newBuilder()
.setOldDirectory(oldParent)
.setOldName(oldName)
.setNewDirectory(newParent)
.setNewName(newName)
.build());
} catch (Exception e) {
LOG.warn("atomicRenameEntry {}/{} => {}/{}: {}", oldParent, oldName, newParent, newName, e);
return false;
}
return true;
}
private FilerProto.Entry fixEntryAfterReading(FilerProto.Entry entry) {
if (entry.getChunksList().size() <= 0) {
return entry;
}
String fileId = entry.getChunks(0).getFileId();
if (fileId != null && fileId.length() != 0) {
return entry;
}
FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
entryBuilder.clearChunks();
for (FilerProto.FileChunk chunk : entry.getChunksList()) {
FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder();
FilerProto.FileId fid = chunk.getFid();
fileId = String.format("%d,%d%x", fid.getVolumeId(), fid.getFileKey(), fid.getCookie());
chunkBuilder.setFileId(fileId);
entryBuilder.addChunks(chunkBuilder);
}
return entryBuilder.build();
}
}

31
other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java
View File

@ -2,7 +2,14 @@ package seaweedfs.client;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
import io.grpc.netty.shaded.io.grpc.netty.NegotiationType;
import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
import javax.net.ssl.SSLException;
import java.io.File;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;
@ -20,6 +27,16 @@ public class FilerGrpcClient {
this(ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext());
}
public FilerGrpcClient(String host, int grpcPort,
String caFilePath,
String clientCertFilePath,
String clientPrivateKeyFilePath) throws SSLException {
this(NettyChannelBuilder.forAddress(host, grpcPort)
.negotiationType(NegotiationType.TLS)
.sslContext(buildSslContext(caFilePath,clientCertFilePath,clientPrivateKeyFilePath)));
}
public FilerGrpcClient(ManagedChannelBuilder<?> channelBuilder) {
channel = channelBuilder.build();
blockingStub = SeaweedFilerGrpc.newBlockingStub(channel);
@ -42,4 +59,18 @@ public class FilerGrpcClient {
public SeaweedFilerGrpc.SeaweedFilerFutureStub getFutureStub() {
return futureStub;
}
private static SslContext buildSslContext(String trustCertCollectionFilePath,
String clientCertChainFilePath,
String clientPrivateKeyFilePath) throws SSLException {
SslContextBuilder builder = GrpcSslContexts.forClient();
if (trustCertCollectionFilePath != null) {
builder.trustManager(new File(trustCertCollectionFilePath));
}
if (clientCertChainFilePath != null && clientPrivateKeyFilePath != null) {
builder.keyManager(new File(clientCertChainFilePath), new File(clientPrivateKeyFilePath));
}
return builder.build();
}
}

7
other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java
View File

@ -29,9 +29,10 @@ public class SeaweedWrite {
.build());
String fileId = response.getFileId();
String url = response.getUrl();
String auth = response.getAuth();
String targetUrl = String.format("http://%s/%s", url, fileId);
String etag = multipartUpload(targetUrl, bytes, bytesOffset, bytesLength);
String etag = multipartUpload(targetUrl, auth, bytes, bytesOffset, bytesLength);
entry.addChunks(FilerProto.FileChunk.newBuilder()
.setFileId(fileId)
@ -54,6 +55,7 @@ public class SeaweedWrite {
}
private static String multipartUpload(String targetUrl,
String auth,
final byte[] bytes,
final long bytesOffset, final long bytesLength) throws IOException {
@ -62,6 +64,9 @@ public class SeaweedWrite {
InputStream inputStream = new ByteArrayInputStream(bytes, (int) bytesOffset, (int) bytesLength);
HttpPost post = new HttpPost(targetUrl);
if (auth != null && auth.length() != 0) {
post.addHeader("Authorization", "BEARER " + auth);
}
post.setEntity(MultipartEntityBuilder.create()
.setMode(HttpMultipartMode.BROWSER_COMPATIBLE)

44
other/java/client/src/main/proto/filer.proto
View File

@ -24,6 +24,9 @@ service SeaweedFiler {
rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
}
rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
}
rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
}
@ -36,6 +39,9 @@ service SeaweedFiler {
rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
}
rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) {
}
}
//////////////////////////////////////////////////
@ -69,19 +75,33 @@ message Entry {
map<string, bytes> extended = 5;
}
message FullEntry {
string dir = 1;
Entry entry = 2;
}
message EventNotification {
Entry old_entry = 1;
Entry new_entry = 2;
bool delete_chunks = 3;
string new_parent_path = 4;
}
message FileChunk {
string file_id = 1;
string file_id = 1; // to be deprecated
int64 offset = 2;
uint64 size = 3;
int64 mtime = 4;
string e_tag = 5;
string source_file_id = 6;
string source_file_id = 6; // to be deprecated
FileId fid = 7;
FileId source_fid = 8;
}
message FileId {
uint32 volume_id = 1;
uint64 file_key = 2;
fixed32 cookie = 3;
}
message FuseAttributes {
@ -126,6 +146,16 @@ message DeleteEntryRequest {
message DeleteEntryResponse {
}
message AtomicRenameEntryRequest {
string old_directory = 1;
string old_name = 2;
string new_directory = 3;
string new_name = 4;
}
message AtomicRenameEntryResponse {
}
message AssignVolumeRequest {
int32 count = 1;
string collection = 2;
@ -139,6 +169,7 @@ message AssignVolumeResponse {
string url = 2;
string public_url = 3;
int32 count = 4;
string auth = 5;
}
message LookupVolumeRequest {
@ -177,3 +208,12 @@ message StatisticsResponse {
uint64 used_size = 5;
uint64 file_count = 6;
}
message GetFilerConfigurationRequest {
}
message GetFilerConfigurationResponse {
repeated string masters = 1;
string replication = 2;
string collection = 3;
uint32 max_mb = 4;
}

2
other/java/hdfs/pom.xml
View File

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<properties>
<seaweedfs.client.version>1.0.5</seaweedfs.client.version>
<seaweedfs.client.version>1.1.0</seaweedfs.client.version>
<hadoop.version>3.1.1</hadoop.version>
</properties>

19
other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
View File

@ -34,6 +34,9 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
public static final String FS_SEAWEED_GRPC_CA = "fs.seaweed.ca";
public static final String FS_SEAWEED_GRPC_CLIENT_KEY = "fs.seaweed.client.key";
public static final String FS_SEAWEED_GRPC_CLIENT_CERT = "fs.seaweed.client.cert";
private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
private static int BUFFER_SIZE = 16 * 1024 * 1024;
@ -72,9 +75,19 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
setConf(conf);
this.uri = uri;
if (conf.get(FS_SEAWEED_GRPC_CA) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CA).length() != 0
&& conf.get(FS_SEAWEED_GRPC_CLIENT_CERT) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_CERT).length() != 0
&& conf.get(FS_SEAWEED_GRPC_CLIENT_KEY) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_KEY).length() != 0) {
seaweedFileSystemStore = new SeaweedFileSystemStore(host, port,
conf.get(FS_SEAWEED_GRPC_CA),
conf.get(FS_SEAWEED_GRPC_CLIENT_CERT),
conf.get(FS_SEAWEED_GRPC_CLIENT_KEY));
} else {
seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
}
}
@Override
public FSDataInputStream open(Path path, int bufferSize) throws IOException {
@ -271,6 +284,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
/**
* Concat existing files together.
*
* @param trg the path to the target destination.
* @param psrcs the paths to the sources to use for the concatenation.
* @throws IOException IO failure
@ -291,9 +305,9 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
* <li>Fails if path is not closed.</li>
* <li>Fails if new size is greater than current size.</li>
* </ul>
*
* @param f The path to the file to be truncated
* @param newLength The size the file is to be truncated to
*
* @return <code>true</code> if the file has been truncated to the desired
* <code>newLength</code> and is immediately available to be reused for
* write operations such as <code>append</code>, or
@ -327,6 +341,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
/**
* Create a snapshot.
*
* @param path The directory where snapshots will be taken.
* @param snapshotName The name of the snapshot
* @return the snapshot path.
@ -342,6 +357,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
/**
* Rename a snapshot.
*
* @param path The directory path where the snapshot was taken
* @param snapshotOldName Old name of the snapshot
* @param snapshotNewName New name of the snapshot
@ -358,6 +374,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
/**
* Delete a snapshot of a directory.
*
* @param path The directory that the to-be-deleted snapshot belongs to
* @param snapshotName The name of the snapshot
* @throws IOException IO failure

38
other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
View File

@ -12,6 +12,7 @@ import seaweedfs.client.FilerGrpcClient;
import seaweedfs.client.FilerProto;
import seaweedfs.client.SeaweedRead;
import javax.net.ssl.SSLException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@ -33,6 +34,13 @@ public class SeaweedFileSystemStore {
filerClient = new FilerClient(filerGrpcClient);
}
public SeaweedFileSystemStore(String host, int port,
String caFile, String clientCertFile, String clientKeyFile) throws SSLException {
int grpcPort = 10000 + port;
filerGrpcClient = new FilerGrpcClient(host, grpcPort, caFile, clientCertFile, clientKeyFile);
filerClient = new FilerClient(filerGrpcClient);
}
public static String getParentDirectory(Path path) {
return path.isRoot() ? "/" : path.getParent().toUri().getPath();
}
@ -143,35 +151,7 @@ public class SeaweedFileSystemStore {
LOG.warn("rename non-existing source: {}", source);
return;
}
LOG.warn("rename moveEntry source: {}", source);
moveEntry(source.getParent(), entry, destination);
}
private boolean moveEntry(Path oldParent, FilerProto.Entry entry, Path destination) {
LOG.debug("moveEntry: {}/{} => {}", oldParent, entry.getName(), destination);
FilerProto.Entry.Builder newEntry = entry.toBuilder().setName(destination.getName());
boolean isDirectoryCreated = filerClient.createEntry(getParentDirectory(destination), newEntry.build());
if (!isDirectoryCreated) {
return false;
}
if (entry.getIsDirectory()) {
Path entryPath = new Path(oldParent, entry.getName());
List<FilerProto.Entry> entries = filerClient.listEntries(entryPath.toUri().getPath());
for (FilerProto.Entry ent : entries) {
boolean isSucess = moveEntry(entryPath, ent, new Path(destination, ent.getName()));
if (!isSucess) {
return false;
}
}
}
return filerClient.deleteEntry(
oldParent.toUri().getPath(), entry.getName(), false, false);
filerClient.mv(source.toUri().getPath(), destination.toUri().getPath());
}
public OutputStream createFile(final Path path,

3
unmaintained/change_superblock/change_superblock.go
View File

@ -9,6 +9,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
)
var (
@ -73,7 +74,7 @@ func main() {
}
if *targetTTL != "" {
ttl, err := storage.ReadTTL(*targetTTL)
ttl, err := needle.ReadTTL(*targetTTL)
if err != nil {
glog.Fatalf("cannot parse target ttl %s: %v", *targetTTL, err)

35
unmaintained/compact_leveldb/compact_leveldb.go Normal file
View File

@ -0,0 +1,35 @@
package main
import (
"flag"
"log"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/util"
)
var (
dir = flag.String("dir", ".", "data directory to store leveldb files")
)
func main() {
flag.Parse()
opts := &opt.Options{
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
CompactionTableSizeMultiplier: 10,
OpenFilesCacheCapacity: -1,
}
db, err := leveldb.OpenFile(*dir, opts)
if err != nil {
log.Fatal(err)
}
defer db.Close()
if err := db.CompactRange(util.Range{}); err != nil {
log.Fatal(err)
}
}

15
unmaintained/fix_dat/fix_dat.go
View File

@ -10,6 +10,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
)
@ -61,7 +62,7 @@ func main() {
}
newDatFile.Write(superBlock.Bytes())
iterateEntries(datFile, indexFile, func(n *storage.Needle, offset int64) {
iterateEntries(datFile, indexFile, func(n *needle.Needle, offset int64) {
fmt.Printf("needle id=%v name=%s size=%d dataSize=%d\n", n.Id, string(n.Name), n.Size, n.DataSize)
_, s, _, e := n.Append(newDatFile, superBlock.Version())
fmt.Printf("size %d error %v\n", s, e)
@ -69,7 +70,7 @@ func main() {
}
func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needle, offset int64)) {
func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *needle.Needle, offset int64)) {
// start to read index file
var readerOffset int64
bytes := make([]byte, 16)
@ -84,7 +85,7 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
}
offset := int64(superBlock.BlockSize())
version := superBlock.Version()
n, rest, err := storage.ReadNeedleHeader(datFile, version, offset)
n, _, rest, err := needle.ReadNeedleHeader(datFile, version, offset)
if err != nil {
fmt.Printf("cannot read needle header: %v", err)
return
@ -106,7 +107,7 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
fmt.Printf("key: %d offsetFromIndex %d n.Size %d sizeFromIndex:%d\n", key, offsetFromIndex, n.Size, sizeFromIndex)
rest = storage.NeedleBodyLength(sizeFromIndex, version)
rest = needle.NeedleBodyLength(sizeFromIndex, version)
func() {
defer func() {
@ -114,7 +115,7 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
fmt.Println("Recovered in f", r)
}
}()
if err = n.ReadNeedleBody(datFile, version, offset+int64(types.NeedleEntrySize), rest); err != nil {
if _, err = n.ReadNeedleBody(datFile, version, offset+int64(types.NeedleHeaderSize), rest); err != nil {
fmt.Printf("cannot read needle body: offset %d body %d %v\n", offset, rest, err)
}
}()
@ -124,9 +125,9 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
}
visitNeedle(n, offset)
offset += types.NeedleEntrySize + rest
offset += types.NeedleHeaderSize + rest
//fmt.Printf("==> new entry offset %d\n", offset)
if n, rest, err = storage.ReadNeedleHeader(datFile, version, offset); err != nil {
if n, _, rest, err = needle.ReadNeedleHeader(datFile, version, offset); err != nil {
if err == io.EOF {
return
}

156
unmaintained/load_test/load_test_leveldb.go Normal file
View File

@ -0,0 +1,156 @@
package main
import (
"crypto/md5"
"flag"
"fmt"
"io"
"log"
"math/rand"
"os"
"sync"
"time"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/opt"
)
var (
dir = flag.String("dir", "./t", "directory to store level db files")
useHash = flag.Bool("isHash", false, "hash the path as the key")
dbCount = flag.Int("dbCount", 1, "the number of leveldb")
)
func main() {
flag.Parse()
totalTenants := 300
totalYears := 3
opts := &opt.Options{
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
CompactionTableSizeMultiplier: 4,
}
var dbs []*leveldb.DB
var chans []chan string
for d := 0 ; d < *dbCount; d++ {
dbFolder := fmt.Sprintf("%s/%02d", *dir, d)
os.MkdirAll(dbFolder, 0755)
db, err := leveldb.OpenFile(dbFolder, opts)
if err != nil {
log.Printf("filer store open dir %s: %v", *dir, err)
return
}
dbs = append(dbs, db)
chans = append(chans, make(chan string, 1024))
}
var wg sync.WaitGroup
for d := 0 ; d < *dbCount; d++ {
wg.Add(1)
go func(d int){
defer wg.Done()
ch := chans[d]
db := dbs[d]
for p := range ch {
if *useHash {
insertAsHash(db, p)
}else{
insertAsFullPath(db, p)
}
}
}(d)
}
counter := int64(0)
lastResetTime := time.Now()
r := rand.New(rand.NewSource(35))
for y := 0; y < totalYears; y++ {
for m := 0; m < 12; m++ {
for d := 0; d < 31; d++ {
for h := 0; h < 24; h++ {
for min := 0; min < 60; min++ {
for i := 0; i < totalTenants; i++ {
p := fmt.Sprintf("tenent%03d/%4d/%02d/%02d/%02d/%02d", i, 2015+y, 1+m, 1+d, h, min)
x := r.Intn(*dbCount)
chans[x] <- p
counter++
}
t := time.Now()
if lastResetTime.Add(time.Second).Before(t) {
p := fmt.Sprintf("%4d/%02d/%02d/%02d/%02d", 2015+y, 1+m, 1+d, h, min)
fmt.Printf("%s = %4d put/sec\n", p, counter)
counter = 0
lastResetTime = t
}
}
}
}
}
}
for d := 0 ; d < *dbCount; d++ {
close(chans[d])
}
wg.Wait()
}
func insertAsFullPath(db *leveldb.DB, p string) {
_, getErr := db.Get([]byte(p), nil)
if getErr == leveldb.ErrNotFound {
putErr := db.Put([]byte(p), []byte(p), nil)
if putErr != nil {
log.Printf("failed to put %s", p)
}
}
}
func insertAsHash(db *leveldb.DB, p string) {
key := fmt.Sprintf("%d:%s", hashToLong(p), p)
_, getErr := db.Get([]byte(key), nil)
if getErr == leveldb.ErrNotFound {
putErr := db.Put([]byte(key), []byte(p), nil)
if putErr != nil {
log.Printf("failed to put %s", p)
}
}
}
func hashToLong(dir string) (v int64) {
h := md5.New()
io.WriteString(h, dir)
b := h.Sum(nil)
v += int64(b[0])
v <<= 8
v += int64(b[1])
v <<= 8
v += int64(b[2])
v <<= 8
v += int64(b[3])
v <<= 8
v += int64(b[4])
v <<= 8
v += int64(b[5])
v <<= 8
v += int64(b[6])
v <<= 8
v += int64(b[7])
return
}

12
unmaintained/repeated_vacuum/repeated_vacuum.go
View File

@ -7,6 +7,9 @@ import (
"log"
"math/rand"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/spf13/viper"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/util"
)
@ -19,8 +22,11 @@ var (
func main() {
flag.Parse()
util.LoadConfiguration("security", false)
grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
for i := 0; i < *repeat; i++ {
assignResult, err := operation.Assign(*master, &operation.VolumeAssignRequest{Count: 1})
assignResult, err := operation.Assign(*master, grpcDialOption, &operation.VolumeAssignRequest{Count: 1})
if err != nil {
log.Fatalf("assign: %v", err)
}
@ -31,12 +37,12 @@ func main() {
targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
_, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), reader, false, "", nil, "")
_, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), reader, false, "", nil, assignResult.Auth)
if err != nil {
log.Fatalf("upload: %v", err)
}
util.Delete(targetUrl, "")
util.Delete(targetUrl, string(assignResult.Auth))
util.Get(fmt.Sprintf("http://%s/vol/vacuum", *master))

15
unmaintained/see_dat/see_dat.go
View File

@ -2,8 +2,12 @@ package main
import (
"flag"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"time"
)
var (
@ -13,7 +17,7 @@ var (
)
type VolumeFileScanner4SeeDat struct {
version storage.Version
version needle.Version
}
func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock storage.SuperBlock) error {
@ -22,18 +26,19 @@ func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock storage.Supe
}
func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
return false
return true
}
func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *storage.Needle, offset int64) error {
glog.V(0).Infof("%d,%s%x offset %d size %d cookie %x", *volumeId, n.Id, n.Cookie, offset, n.Size, n.Cookie)
func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64) error {
t := time.Unix(int64(n.AppendAtNs)/int64(time.Second), int64(n.AppendAtNs)%int64(time.Second))
glog.V(0).Infof("%d,%s%x offset %d size %d cookie %x appendedAt %v", *volumeId, n.Id, n.Cookie, offset, n.Size, n.Cookie, t)
return nil
}
func main() {
flag.Parse()
vid := storage.VolumeId(*volumeId)
vid := needle.VolumeId(*volumeId)
scanner := &VolumeFileScanner4SeeDat{}
err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid, storage.NeedleMapInMemory, scanner)

4
unmaintained/see_idx/see_idx.go
View File

@ -8,7 +8,7 @@ import (
"strconv"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/idx"
"github.com/chrislusf/seaweedfs/weed/storage/types"
)
@ -35,7 +35,7 @@ func main() {
}
defer indexFile.Close()
storage.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
idx.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
fmt.Printf("key:%v offset:%v size:%v\n", key, offset, size)
return nil
})

68
unmaintained/see_meta/see_meta.go Normal file
View File

@ -0,0 +1,68 @@
package main
import (
"flag"
"fmt"
"io"
"log"
"os"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/golang/protobuf/proto"
)
var (
metaFile = flag.String("meta", "", "meta file generated via fs.meta.save")
)
func main() {
flag.Parse()
dst, err := os.OpenFile(*metaFile, os.O_RDONLY, 0644)
if err != nil {
log.Fatalf("failed to open %s: %v", *metaFile, err)
}
defer dst.Close()
err = walkMetaFile(dst)
if err != nil {
log.Fatalf("failed to visit %s: %v", *metaFile, err)
}
}
func walkMetaFile(dst *os.File) error {
sizeBuf := make([]byte, 4)
for {
if n, err := dst.Read(sizeBuf); n != 4 {
if err == io.EOF {
return nil
}
return err
}
size := util.BytesToUint32(sizeBuf)
data := make([]byte, int(size))
if n, err := dst.Read(data); n != len(data) {
return err
}
fullEntry := &filer_pb.FullEntry{}
if err := proto.Unmarshal(data, fullEntry); err != nil {
return err
}
fmt.Fprintf(os.Stdout, "file %s %v\n", filer2.FullPath(fullEntry.Dir).Child(fullEntry.Entry.Name), fullEntry.Entry.Attributes.String())
for i, chunk := range fullEntry.Entry.Chunks {
fmt.Fprintf(os.Stdout, " chunk %d %v\n", i+1, chunk.String())
}
}
}

136
unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go Normal file
View File

@ -0,0 +1,136 @@
package main
import (
"bytes"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"math/rand"
"mime/multipart"
"net/http"
"os"
"strings"
"sync"
"time"
)
var (
size = flag.Int("size", 1024, "file size")
concurrency = flag.Int("c", 4, "concurrent number of uploads")
times = flag.Int("n", 1024, "repeated number of times")
fileCount = flag.Int("fileCount", 1, "number of files to write")
destination = flag.String("to", "http://localhost:8888/", "destination directory on filer")
statsChan = make(chan stat, 8)
)
type stat struct {
size int64
}
func main() {
flag.Parse()
data := make([]byte, *size)
println("data len", len(data))
var wg sync.WaitGroup
for x := 0; x < *concurrency; x++ {
wg.Add(1)
go func(x int) {
defer wg.Done()
client := &http.Client{Transport: &http.Transport{
MaxConnsPerHost: 1024,
MaxIdleConnsPerHost: 1024,
}}
r := rand.New(rand.NewSource(time.Now().UnixNano() + int64(x)))
for t := 0; t < *times; t++ {
for f := 0; f < *fileCount; f++ {
fn := r.Intn(*fileCount)
if size, err := uploadFileToFiler(client, data, fmt.Sprintf("file%04d", fn), *destination); err == nil {
statsChan <- stat{
size: size,
}
} else {
log.Fatalf("client %d upload %d times: %v", x, t, err)
}
}
}
}(x)
}
go func() {
ticker := time.NewTicker(1000 * time.Millisecond)
var lastTime time.Time
var counter, size int64
for {
select {
case stat := <-statsChan:
size += stat.size
counter++
case x := <-ticker.C:
if !lastTime.IsZero() {
elapsed := x.Sub(lastTime).Seconds()
fmt.Fprintf(os.Stdout, "%.2f files/s, %.2f MB/s\n",
float64(counter)/elapsed,
float64(size/1024/1024)/elapsed)
}
lastTime = x
size = 0
counter = 0
}
}
}()
wg.Wait()
}
func uploadFileToFiler(client *http.Client, data []byte, filename, destination string) (size int64, err error) {
if !strings.HasSuffix(destination, "/") {
destination = destination + "/"
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", filename)
if err != nil {
return 0, fmt.Errorf("fail to create form %v: %v", filename, err)
}
part.Write(data)
err = writer.Close()
if err != nil {
return 0, fmt.Errorf("fail to write part %v: %v", filename, err)
}
uri := destination + filename
request, err := http.NewRequest("POST", uri, body)
request.Header.Set("Content-Type", writer.FormDataContentType())
// request.Close = true // can not use this, which do not reuse http connection, impacting filer->volume also.
resp, err := client.Do(request)
if err != nil {
return 0, fmt.Errorf("http POST %s: %v", uri, err)
} else {
body := &bytes.Buffer{}
_, err := body.ReadFrom(resp.Body)
if err != nil {
return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
}
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
}
return int64(len(data)), nil
}

150
unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go Normal file
View File

@ -0,0 +1,150 @@
package main
import (
"bytes"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"math/rand"
"mime/multipart"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"time"
)
var (
dir = flag.String("dir", ".", "upload files under this directory")
concurrency = flag.Int("c", 1, "concurrent number of uploads")
times = flag.Int("n", 1, "repeated number of times")
destination = flag.String("to", "http://localhost:8888/", "destination directory on filer")
statsChan = make(chan stat, 8)
)
type stat struct {
size int64
}
func main() {
flag.Parse()
var fileNames []string
files, err := ioutil.ReadDir(*dir)
if err != nil {
log.Fatalf("fail to read dir %v: %v", *dir, err)
}
for _, file := range files {
if file.IsDir() {
continue
}
fileNames = append(fileNames, filepath.Join(*dir, file.Name()))
}
var wg sync.WaitGroup
for x := 0; x < *concurrency; x++ {
wg.Add(1)
client := &http.Client{}
go func() {
defer wg.Done()
rand.Shuffle(len(fileNames), func(i, j int) {
fileNames[i], fileNames[j] = fileNames[j], fileNames[i]
})
for t := 0; t < *times; t++ {
for _, filename := range fileNames {
if size, err := uploadFileToFiler(client, filename, *destination); err == nil {
statsChan <- stat{
size: size,
}
}
}
}
}()
}
go func() {
ticker := time.NewTicker(500 * time.Millisecond)
var lastTime time.Time
var counter, size int64
for {
select {
case stat := <-statsChan:
size += stat.size
counter++
case x := <-ticker.C:
if !lastTime.IsZero() {
elapsed := x.Sub(lastTime).Seconds()
fmt.Fprintf(os.Stdout, "%.2f files/s, %.2f MB/s\n",
float64(counter)/elapsed,
float64(size/1024/1024)/elapsed)
}
lastTime = x
size = 0
counter = 0
}
}
}()
wg.Wait()
}
func uploadFileToFiler(client *http.Client, filename, destination string) (size int64, err error) {
file, err := os.Open(filename)
if err != nil {
panic(err)
}
defer file.Close()
fi, err := file.Stat()
if !strings.HasSuffix(destination, "/") {
destination = destination + "/"
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", file.Name())
if err != nil {
return 0, fmt.Errorf("fail to create form %v: %v", file.Name(), err)
}
_, err = io.Copy(part, file)
if err != nil {
return 0, fmt.Errorf("fail to write part %v: %v", file.Name(), err)
}
err = writer.Close()
if err != nil {
return 0, fmt.Errorf("fail to write part %v: %v", file.Name(), err)
}
uri := destination + file.Name()
request, err := http.NewRequest("POST", uri, body)
request.Header.Set("Content-Type", writer.FormDataContentType())
resp, err := client.Do(request)
if err != nil {
return 0, fmt.Errorf("http POST %s: %v", uri, err)
} else {
body := &bytes.Buffer{}
_, err := body.ReadFrom(resp.Body)
if err != nil {
return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
}
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
}
return fi.Size(), nil
}

70
unmaintained/volume_tailer/volume_tailer.go Normal file
View File

@ -0,0 +1,70 @@
package main
import (
"flag"
"log"
"time"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
util2 "github.com/chrislusf/seaweedfs/weed/util"
"github.com/spf13/viper"
"golang.org/x/tools/godoc/util"
)
var (
master = flag.String("master", "localhost:9333", "master server host and port")
volumeId = flag.Int("volumeId", -1, "a volume id")
rewindDuration = flag.Duration("rewind", -1, "rewind back in time. -1 means from the first entry. 0 means from now.")
timeoutSeconds = flag.Int("timeoutSeconds", 0, "disconnect if no activity after these seconds")
showTextFile = flag.Bool("showTextFile", false, "display textual file content")
)
func main() {
flag.Parse()
util2.LoadConfiguration("security", false)
grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
vid := needle.VolumeId(*volumeId)
var sinceTimeNs int64
if *rewindDuration == 0 {
sinceTimeNs = time.Now().UnixNano()
} else if *rewindDuration == -1 {
sinceTimeNs = 0
} else if *rewindDuration > 0 {
sinceTimeNs = time.Now().Add(-*rewindDuration).UnixNano()
}
err := operation.TailVolume(*master, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) {
if n.Size == 0 {
println("-", n.String())
return nil
} else {
println("+", n.String())
}
if *showTextFile {
data := n.Data
if n.IsGzipped() {
if data, err = util2.UnGzipData(data); err != nil {
return err
}
}
if util.IsText(data) {
println(string(data))
}
println("-", n.String(), "compressed", n.IsGzipped(), "original size", len(data))
}
return nil
})
if err != nil {
log.Printf("Error VolumeTailSender volume %d: %v", vid, err)
}
}

98
util/gostd Executable file
View File

@ -0,0 +1,98 @@
#!/usr/bin/env bash
############################ GLOBAL VARIABLES
regex=' '
branch="master"
max_length=150
REGEX_SUFFIX_GO=".+\.go$"
############################ FUNCTIONS
msg() {
printf '%b' "$1" >&2
}
die() {
msg "\33[31m[✘]\33[0m ${1}${2}"
exit 1
}
succ() {
msg "\33[34m[√]\33[0m ${1}${2}"
}
gostd() {
local branch=$1
local reg4exclude=$2
local max_length=$3
for file in `git diff $branch --name-only`
do
if ! [[ $file =~ $REGEX_SUFFIX_GO ]] || [[ $file =~ $reg4exclude ]]; then
continue
fi
error=`go fmt $file 2>&1`
if ! [ $? -eq 0 ]; then
die "go fmt $file:" "$error"
fi
succ "$file\n"
grep -n -E --color=always ".{$max_length}" $file | awk '{ printf ("%4s %s\n", "", $0) }'
done
}
get_options() {
while getopts "b:e:hl:" opts
do
case $opts in
b)
branch=$OPTARG
;;
e)
regex=$OPTARG
;;
h)
usage
exit 0
;;
l)
max_length=$OPTARG
;;
\?)
usage
exit 1
;;
esac
done
}
usage () {
cat << _EOC_
Usage:
gostd [options]
Options:
-b <branch/commit> Specify the git diff branch or commit.
(default: master)
-e <regex> Regex for excluding file or directory.
-h Print this usage.
-l <length> Show files that exceed the limit line length.
(default: 150)
Examples:
gostd
gostd -b master -l 100
gostd -b 59d532a -e weed/pb -l 100
_EOC_
}
main() {
get_options "$@"
gostd "$branch" "$regex" "$max_length"
}
############################ MAIN()
main "$@"

46
weed/command/backup.go
View File

@ -3,6 +3,11 @@ package command
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/spf13/viper"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/storage"
)
@ -41,15 +46,19 @@ var cmdBackup = &Command{
But it's tricky to incrementally copy the differences.
The complexity comes when there are multiple addition, deletion and compaction.
This tool will handle them correctly and efficiently, avoiding unnecessary data transporation.
This tool will handle them correctly and efficiently, avoiding unnecessary data transportation.
`,
}
func runBackup(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false)
grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
if *s.volumeId == -1 {
return false
}
vid := storage.VolumeId(*s.volumeId)
vid := needle.VolumeId(*s.volumeId)
// find volume location, replication, ttl info
lookup, err := operation.Lookup(*s.master, vid.String())
@ -59,12 +68,12 @@ func runBackup(cmd *Command, args []string) bool {
}
volumeServer := lookup.Locations[0].Url
stats, err := operation.GetVolumeSyncStatus(volumeServer, uint32(vid))
stats, err := operation.GetVolumeSyncStatus(volumeServer, grpcDialOption, uint32(vid))
if err != nil {
fmt.Printf("Error get volume %d status: %v\n", vid, err)
return true
}
ttl, err := storage.ReadTTL(stats.Ttl)
ttl, err := needle.ReadTTL(stats.Ttl)
if err != nil {
fmt.Printf("Error get volume %d ttl %s: %v\n", vid, stats.Ttl, err)
return true
@ -81,7 +90,34 @@ func runBackup(cmd *Command, args []string) bool {
return true
}
if err := v.Synchronize(volumeServer); err != nil {
if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
if err = v.Compact(0, 0); err != nil {
fmt.Printf("Compact Volume before synchronizing %v\n", err)
return true
}
if err = v.CommitCompact(); err != nil {
fmt.Printf("Commit Compact before synchronizing %v\n", err)
return true
}
v.SuperBlock.CompactionRevision = uint16(stats.CompactRevision)
v.DataFile().WriteAt(v.SuperBlock.Bytes(), 0)
}
datSize, _, _ := v.FileStat()
if datSize > stats.TailOffset {
// remove the old data
v.Destroy()
// recreate an empty volume
v, err = storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0)
if err != nil {
fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
return true
}
}
defer v.Close()
if err := v.IncrementalBackup(volumeServer, grpcDialOption); err != nil {
fmt.Printf("Error synchronizing volume %d: %v\n", vid, err)
return true
}

39
weed/command/benchmark.go
View File

@ -15,6 +15,9 @@ import (
"sync"
"time"
"github.com/spf13/viper"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/security"
@ -33,15 +36,17 @@ type BenchmarkOptions struct {
read *bool
sequentialRead *bool
collection *string
replication *string
cpuprofile *string
maxCpu *int
secretKey *string
grpcDialOption grpc.DialOption
masterClient *wdclient.MasterClient
}
var (
b BenchmarkOptions
sharedBytes []byte
masterClient *wdclient.MasterClient
isSecure bool
)
func init() {
@ -57,9 +62,9 @@ func init() {
b.read = cmdBenchmark.Flag.Bool("read", true, "enable read")
b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file")
b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection")
b.replication = cmdBenchmark.Flag.String("replication", "000", "replication type")
b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file")
b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
b.secretKey = cmdBenchmark.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
sharedBytes = make([]byte, 1024)
}
@ -102,6 +107,10 @@ var (
)
func runBenchmark(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false)
b.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
if *b.maxCpu < 1 {
*b.maxCpu = runtime.NumCPU()
@ -116,9 +125,9 @@ func runBenchmark(cmd *Command, args []string) bool {
defer pprof.StopCPUProfile()
}
masterClient = wdclient.NewMasterClient(context.Background(), "benchmark", strings.Split(*b.masters, ","))
go masterClient.KeepConnectedToMaster()
masterClient.WaitUntilConnected()
b.masterClient = wdclient.NewMasterClient(context.Background(), b.grpcDialOption, "client", strings.Split(*b.masters, ","))
go b.masterClient.KeepConnectedToMaster()
b.masterClient.WaitUntilConnected()
if *b.write {
benchWrite()
@ -188,7 +197,6 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
defer wait.Done()
delayedDeleteChan := make(chan *delayedFile, 100)
var waitForDeletions sync.WaitGroup
secret := security.Secret(*b.secretKey)
for i := 0; i < 7; i++ {
waitForDeletions.Add(1)
@ -198,8 +206,11 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
if df.enterTime.After(time.Now()) {
time.Sleep(df.enterTime.Sub(time.Now()))
}
if e := util.Delete("http://"+df.fp.Server+"/"+df.fp.Fid,
security.GenJwt(secret, df.fp.Fid)); e == nil {
var jwtAuthorization security.EncodedJwt
if isSecure {
jwtAuthorization = operation.LookupJwt(b.masterClient.GetMaster(), df.fp.Fid)
}
if e := util.Delete(fmt.Sprintf("http://%s/%s", df.fp.Server, df.fp.Fid), string(jwtAuthorization)); e == nil {
s.completed++
} else {
s.failed++
@ -221,10 +232,14 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
ar := &operation.VolumeAssignRequest{
Count: 1,
Collection: *b.collection,
Replication: *b.replication,
}
if assignResult, err := operation.Assign(masterClient.GetMaster(), ar); err == nil {
if assignResult, err := operation.Assign(b.masterClient.GetMaster(), b.grpcDialOption, ar); err == nil {
fp.Server, fp.Fid, fp.Collection = assignResult.Url, assignResult.Fid, *b.collection
if _, err := fp.Upload(0, masterClient.GetMaster(), secret); err == nil {
if !isSecure && assignResult.Auth != "" {
isSecure = true
}
if _, err := fp.Upload(0, b.masterClient.GetMaster(), assignResult.Auth, b.grpcDialOption); err == nil {
if random.Intn(100) < *b.deletePercentage {
s.total++
delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp}
@ -264,7 +279,7 @@ func readFiles(fileIdLineChan chan string, s *stat) {
fmt.Printf("reading file %s\n", fid)
}
start := time.Now()
url, err := masterClient.LookupFileId(fid)
url, err := b.masterClient.LookupFileId(fid)
if err != nil {
s.failed++
println("!!!! ", fid, " location not found!!!!!")

2
weed/command/command.go
View File

@ -13,7 +13,6 @@ var Commands = []*Command{
cmdCompact,
cmdCopy,
cmdFix,
cmdFilerExport,
cmdFilerReplicate,
cmdServer,
cmdMaster,
@ -27,6 +26,7 @@ var Commands = []*Command{
cmdVolume,
cmdExport,
cmdMount,
cmdWebDav,
}
type Command struct {

5
weed/command/compact.go
View File

@ -3,6 +3,7 @@ package command
import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
)
func init() {
@ -35,14 +36,14 @@ func runCompact(cmd *Command, args []string) bool {
preallocate := *compactVolumePreallocate * (1 << 20)
vid := storage.VolumeId(*compactVolumeId)
vid := needle.VolumeId(*compactVolumeId)
v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid,
storage.NeedleMapInMemory, nil, nil, preallocate)
if err != nil {
glog.Fatalf("Load Volume [ERROR] %s\n", err)
}
if *compactMethod == 0 {
if err = v.Compact(preallocate); err != nil {
if err = v.Compact(preallocate, 0); err != nil {
glog.Fatalf("Compact Volume [ERROR] %s\n", err)
}
} else {

24
weed/command/export.go
View File

@ -12,10 +12,12 @@ import (
"text/template"
"time"
"io"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"io"
)
const (
@ -66,10 +68,10 @@ var (
localLocation, _ = time.LoadLocation("Local")
)
func printNeedle(vid storage.VolumeId, n *storage.Needle, version storage.Version, deleted bool) {
key := storage.NewFileIdFromNeedle(vid, n).String()
func printNeedle(vid needle.VolumeId, n *needle.Needle, version needle.Version, deleted bool) {
key := needle.NewFileIdFromNeedle(vid, n).String()
size := n.DataSize
if version == storage.Version1 {
if version == needle.Version1 {
size = n.Size
}
fmt.Printf("%s\t%s\t%d\t%t\t%s\t%s\t%s\t%t\n",
@ -85,10 +87,10 @@ func printNeedle(vid storage.VolumeId, n *storage.Needle, version storage.Versio
}
type VolumeFileScanner4Export struct {
version storage.Version
version needle.Version
counter int
needleMap *storage.NeedleMap
vid storage.VolumeId
vid needle.VolumeId
}
func (scanner *VolumeFileScanner4Export) VisitSuperBlock(superBlock storage.SuperBlock) error {
@ -100,14 +102,14 @@ func (scanner *VolumeFileScanner4Export) ReadNeedleBody() bool {
return true
}
func (scanner *VolumeFileScanner4Export) VisitNeedle(n *storage.Needle, offset int64) error {
func (scanner *VolumeFileScanner4Export) VisitNeedle(n *needle.Needle, offset int64) error {
needleMap := scanner.needleMap
vid := scanner.vid
nv, ok := needleMap.Get(n.Id)
glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped(), ok, nv)
if ok && nv.Size > 0 && int64(nv.Offset)*types.NeedlePaddingSize == offset {
if ok && nv.Size > 0 && nv.Size != types.TombstoneFileSize && nv.Offset.ToAcutalOffset() == offset {
if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
n.LastModified, newerThanUnix)
@ -189,7 +191,7 @@ func runExport(cmd *Command, args []string) bool {
if *export.collection != "" {
fileName = *export.collection + "_" + fileName
}
vid := storage.VolumeId(*export.volumeId)
vid := needle.VolumeId(*export.volumeId)
indexFile, err := os.OpenFile(path.Join(*export.dir, fileName+".idx"), os.O_RDONLY, 0644)
if err != nil {
glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
@ -225,8 +227,8 @@ type nameParams struct {
Ext string
}
func writeFile(vid storage.VolumeId, n *storage.Needle) (err error) {
key := storage.NewFileIdFromNeedle(vid, n).String()
func writeFile(vid needle.VolumeId, n *needle.Needle) (err error) {
key := needle.NewFileIdFromNeedle(vid, n).String()
fileNameTemplateBuffer.Reset()
if err = fileNameTemplate.Execute(fileNameTemplateBuffer,
nameParams{

33
weed/command/filer.go
View File

@ -6,6 +6,9 @@ import (
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/spf13/viper"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/server"
@ -21,17 +24,16 @@ type FilerOptions struct {
masters *string
ip *string
port *int
grpcPort *int
publicPort *int
collection *string
defaultReplicaPlacement *string
redirectOnRead *bool
disableDirListing *bool
maxMB *int
secretKey *string
dirListingLimit *int
dataCenter *string
enableNotification *bool
disableHttp *bool
// default leveldb directory, used in "weed server" mode
defaultLevelDbDirectory *string
@ -43,15 +45,14 @@ func init() {
f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection")
f.ip = cmdFiler.Flag.String("ip", "", "filer server http listen ip address")
f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
f.grpcPort = cmdFiler.Flag.Int("port.grpc", 0, "filer grpc server listen port, default to http port + 10000")
f.publicPort = cmdFiler.Flag.Int("port.public", 0, "port opened to public")
f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified")
f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center")
f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
}
var cmdFiler = &Command{
@ -70,13 +71,15 @@ var cmdFiler = &Command{
The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", or "/etc/seaweedfs/", in that order.
The example filer.toml configuration file can be generated by "weed scaffold filer"
The example filer.toml configuration file can be generated by "weed scaffold -config=filer"
`,
}
func runFiler(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false)
f.startFiler()
return true
@ -91,22 +94,23 @@ func (fo *FilerOptions) startFiler() {
publicVolumeMux = http.NewServeMux()
}
defaultLevelDbDirectory := "./filerdb"
defaultLevelDbDirectory := "./filerldb2"
if fo.defaultLevelDbDirectory != nil {
defaultLevelDbDirectory = *fo.defaultLevelDbDirectory + "/filerdb"
defaultLevelDbDirectory = *fo.defaultLevelDbDirectory + "/filerldb2"
}
fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
Masters: strings.Split(*f.masters, ","),
Masters: strings.Split(*fo.masters, ","),
Collection: *fo.collection,
DefaultReplication: *fo.defaultReplicaPlacement,
RedirectOnRead: *fo.redirectOnRead,
DisableDirListing: *fo.disableDirListing,
MaxMB: *fo.maxMB,
SecretKey: *fo.secretKey,
DirListingLimit: *fo.dirListingLimit,
DataCenter: *fo.dataCenter,
DefaultLevelDbDir: defaultLevelDbDirectory,
DisableHttp: *fo.disableHttp,
Port: *fo.port,
})
if nfs_err != nil {
glog.Fatalf("Filer startup error: %v", nfs_err)
@ -128,7 +132,7 @@ func (fo *FilerOptions) startFiler() {
glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.VERSION, *fo.ip, *fo.port)
filerListener, e := util.NewListener(
":"+strconv.Itoa(*fo.port),
*fo.ip+":"+strconv.Itoa(*fo.port),
time.Duration(10)*time.Second,
)
if e != nil {
@ -136,15 +140,12 @@ func (fo *FilerOptions) startFiler() {
}
// starting grpc server
grpcPort := *fo.grpcPort
if grpcPort == 0 {
grpcPort = *fo.port + 10000
}
grpcPort := *fo.port + 10000
grpcL, err := util.NewListener(":"+strconv.Itoa(grpcPort), 0)
if err != nil {
glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
}
grpcS := util.NewGrpcServer()
grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "filer"))
filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
reflection.Register(grpcS)
go grpcS.Serve(grpcL)

299
weed/command/filer_copy.go
View File

@ -1,52 +1,56 @@
package command
import (
"context"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"context"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"io"
"net/http"
"strconv"
"time"
"github.com/chrislusf/seaweedfs/weed/wdclient"
"github.com/spf13/viper"
"google.golang.org/grpc"
)
var (
copy CopyOptions
waitGroup sync.WaitGroup
)
type CopyOptions struct {
filerGrpcPort *int
master *string
include *string
replication *string
collection *string
ttl *string
maxMB *int
secretKey *string
secret security.Secret
masterClient *wdclient.MasterClient
concurrency *int
compressionLevel *int
grpcDialOption grpc.DialOption
masters []string
}
func init() {
cmdCopy.Run = runCopy // break init cycle
cmdCopy.IsDebug = cmdCopy.Flag.Bool("debug", false, "verbose debug information")
copy.master = cmdCopy.Flag.String("master", "localhost:9333", "SeaweedFS master location")
copy.include = cmdCopy.Flag.String("include", "", "pattens of files to copy, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
copy.replication = cmdCopy.Flag.String("replication", "", "replication type")
copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
copy.maxMB = cmdCopy.Flag.Int("maxMB", 0, "split files larger than the limit")
copy.filerGrpcPort = cmdCopy.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to filer port + 10000")
copy.secretKey = cmdCopy.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit")
copy.concurrency = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
copy.compressionLevel = cmdCopy.Flag.Int("compressionLevel", 9, "local file compression level 1 ~ 9")
}
var cmdCopy = &Command{
@ -66,7 +70,9 @@ var cmdCopy = &Command{
}
func runCopy(cmd *Command, args []string) bool {
copy.secret = security.Secret(*copy.secretKey)
util.LoadConfiguration("security", false)
if len(args) <= 1 {
return false
}
@ -96,67 +102,170 @@ func runCopy(cmd *Command, args []string) bool {
}
filerGrpcPort := filerPort + 10000
if *copy.filerGrpcPort != 0 {
filerGrpcPort = uint64(*copy.filerGrpcPort)
}
filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort)
copy.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
for _, fileOrDir := range fileOrDirs {
if !doEachCopy(fileOrDir, filerUrl.Host, filerGrpcAddress, urlPath) {
ctx := context.Background()
masters, collection, replication, maxMB, err := readFilerConfiguration(ctx, copy.grpcDialOption, filerGrpcAddress)
if err != nil {
fmt.Printf("read from filer %s: %v\n", filerGrpcAddress, err)
return false
}
if *copy.collection == "" {
*copy.collection = collection
}
if *copy.replication == "" {
*copy.replication = replication
}
if *copy.maxMB == 0 {
*copy.maxMB = int(maxMB)
}
copy.masters = masters
copy.masterClient = wdclient.NewMasterClient(ctx, copy.grpcDialOption, "client", copy.masters)
go copy.masterClient.KeepConnectedToMaster()
copy.masterClient.WaitUntilConnected()
if *cmdCopy.IsDebug {
util.SetupProfiling("filer.copy.cpu.pprof", "filer.copy.mem.pprof")
}
fileCopyTaskChan := make(chan FileCopyTask, *copy.concurrency)
go func() {
defer close(fileCopyTaskChan)
for _, fileOrDir := range fileOrDirs {
if err := genFileCopyTask(fileOrDir, urlPath, fileCopyTaskChan); err != nil {
fmt.Fprintf(os.Stderr, "gen file list error: %v\n", err)
break
}
}
}()
for i := 0; i < *copy.concurrency; i++ {
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
worker := FileCopyWorker{
options: &copy,
filerHost: filerUrl.Host,
filerGrpcAddress: filerGrpcAddress,
}
if err := worker.copyFiles(ctx, fileCopyTaskChan); err != nil {
fmt.Fprintf(os.Stderr, "copy file error: %v\n", err)
return
}
}()
}
waitGroup.Wait()
return true
}
func doEachCopy(fileOrDir string, filerAddress, filerGrpcAddress string, path string) bool {
f, err := os.Open(fileOrDir)
func readFilerConfiguration(ctx context.Context, grpcDialOption grpc.DialOption, filerGrpcAddress string) (masters []string, collection, replication string, maxMB uint32, err error) {
err = withFilerClient(ctx, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.GetFilerConfiguration(ctx, &filer_pb.GetFilerConfigurationRequest{})
if err != nil {
fmt.Printf("Failed to open file %s: %v\n", fileOrDir, err)
return false
return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
}
masters, collection, replication, maxMB = resp.Masters, resp.Collection, resp.Replication, resp.MaxMb
return nil
})
return
}
defer f.Close()
fi, err := f.Stat()
func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan FileCopyTask) error {
fi, err := os.Stat(fileOrDir)
if err != nil {
fmt.Printf("Failed to get stat for file %s: %v\n", fileOrDir, err)
return false
fmt.Fprintf(os.Stderr, "Failed to get stat for file %s: %v\n", fileOrDir, err)
return nil
}
mode := fi.Mode()
if mode.IsDir() {
files, _ := ioutil.ReadDir(fileOrDir)
for _, subFileOrDir := range files {
if !doEachCopy(fileOrDir+"/"+subFileOrDir.Name(), filerAddress, filerGrpcAddress, path+fi.Name()+"/") {
return false
if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), destPath+fi.Name()+"/", fileCopyTaskChan); err != nil {
return err
}
}
return true
return nil
}
uid, gid := util.GetFileUidGid(fi)
fileCopyTaskChan <- FileCopyTask{
sourceLocation: fileOrDir,
destinationUrlPath: destPath,
fileSize: fi.Size(),
fileMode: fi.Mode(),
uid: uid,
gid: gid,
}
return nil
}
type FileCopyWorker struct {
options *CopyOptions
filerHost string
filerGrpcAddress string
}
func (worker *FileCopyWorker) copyFiles(ctx context.Context, fileCopyTaskChan chan FileCopyTask) error {
for task := range fileCopyTaskChan {
if err := worker.doEachCopy(ctx, task); err != nil {
return err
}
}
return nil
}
type FileCopyTask struct {
sourceLocation string
destinationUrlPath string
fileSize int64
fileMode os.FileMode
uid uint32
gid uint32
}
func (worker *FileCopyWorker) doEachCopy(ctx context.Context, task FileCopyTask) error {
f, err := os.Open(task.sourceLocation)
if err != nil {
fmt.Printf("Failed to open file %s: %v\n", task.sourceLocation, err)
if _, ok := err.(*os.PathError); ok {
fmt.Printf("skipping %s\n", task.sourceLocation)
return nil
}
return err
}
defer f.Close()
// this is a regular file
if *copy.include != "" {
if ok, _ := filepath.Match(*copy.include, filepath.Base(fileOrDir)); !ok {
return true
if *worker.options.include != "" {
if ok, _ := filepath.Match(*worker.options.include, filepath.Base(task.sourceLocation)); !ok {
return nil
}
}
// find the chunk count
chunkSize := int64(*copy.maxMB * 1024 * 1024)
chunkSize := int64(*worker.options.maxMB * 1024 * 1024)
chunkCount := 1
if chunkSize > 0 && fi.Size() > chunkSize {
chunkCount = int(fi.Size()/chunkSize) + 1
if chunkSize > 0 && task.fileSize > chunkSize {
chunkCount = int(task.fileSize/chunkSize) + 1
}
if chunkCount == 1 {
return uploadFileAsOne(filerAddress, filerGrpcAddress, path, f, fi)
return worker.uploadFileAsOne(ctx, task, f)
}
return uploadFileInChunks(filerAddress, filerGrpcAddress, path, f, fi, chunkCount, chunkSize)
return worker.uploadFileInChunks(ctx, task, f, chunkCount, chunkSize)
}
func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f *os.File, fi os.FileInfo) bool {
func (worker *FileCopyWorker) uploadFileAsOne(ctx context.Context, task FileCopyTask, f *os.File) error {
// upload the file content
fileName := filepath.Base(f.Name())
@ -164,29 +273,27 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
var chunks []*filer_pb.FileChunk
if fi.Size() > 0 {
if task.fileSize > 0 {
// assign a volume
assignResult, err := operation.Assign(*copy.master, &operation.VolumeAssignRequest{
assignResult, err := operation.Assign(worker.options.masterClient.GetMaster(), worker.options.grpcDialOption, &operation.VolumeAssignRequest{
Count: 1,
Replication: *copy.replication,
Collection: *copy.collection,
Ttl: *copy.ttl,
Replication: *worker.options.replication,
Collection: *worker.options.collection,
Ttl: *worker.options.ttl,
})
if err != nil {
fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err)
fmt.Printf("Failed to assign from %v: %v\n", worker.options.masters, err)
}
targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
uploadResult, err := operation.Upload(targetUrl, fileName, f, false, mimeType, nil, "")
uploadResult, err := operation.UploadWithLocalCompressionLevel(targetUrl, fileName, f, false, mimeType, nil, assignResult.Auth, *worker.options.compressionLevel)
if err != nil {
fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
return false
return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
}
if uploadResult.Error != "" {
fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
return false
return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
}
fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)
@ -198,43 +305,42 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
ETag: uploadResult.ETag,
})
fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerAddress, urlFolder, fileName)
fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName)
}
if err := withFilerClient(filerGrpcAddress, func(client filer_pb.SeaweedFilerClient) error {
if err := withFilerClient(ctx, worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.CreateEntryRequest{
Directory: urlFolder,
Directory: task.destinationUrlPath,
Entry: &filer_pb.Entry{
Name: fileName,
Attributes: &filer_pb.FuseAttributes{
Crtime: time.Now().Unix(),
Mtime: time.Now().Unix(),
Gid: uint32(os.Getgid()),
Uid: uint32(os.Getuid()),
FileSize: uint64(fi.Size()),
FileMode: uint32(fi.Mode()),
Gid: task.gid,
Uid: task.uid,
FileSize: uint64(task.fileSize),
FileMode: uint32(task.fileMode),
Mime: mimeType,
Replication: *copy.replication,
Collection: *copy.collection,
TtlSec: int32(util.ParseInt(*copy.ttl, 0)),
Replication: *worker.options.replication,
Collection: *worker.options.collection,
TtlSec: int32(util.ParseInt(*worker.options.ttl, 0)),
},
Chunks: chunks,
},
}
if _, err := client.CreateEntry(context.Background(), request); err != nil {
if _, err := client.CreateEntry(ctx, request); err != nil {
return fmt.Errorf("update fh: %v", err)
}
return nil
}); err != nil {
fmt.Printf("upload data %v to http://%s%s%s: %v\n", fileName, filerAddress, urlFolder, fileName, err)
return false
return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
}
return true
return nil
}
func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string, f *os.File, fi os.FileInfo, chunkCount int, chunkSize int64) bool {
func (worker *FileCopyWorker) uploadFileInChunks(ctx context.Context, task FileCopyTask, f *os.File, chunkCount int, chunkSize int64) error {
fileName := filepath.Base(f.Name())
mimeType := detectMimeType(f)
@ -244,14 +350,14 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
for i := int64(0); i < int64(chunkCount); i++ {
// assign a volume
assignResult, err := operation.Assign(*copy.master, &operation.VolumeAssignRequest{
assignResult, err := operation.Assign(worker.options.masterClient.GetMaster(), worker.options.grpcDialOption, &operation.VolumeAssignRequest{
Count: 1,
Replication: *copy.replication,
Collection: *copy.collection,
Ttl: *copy.ttl,
Replication: *worker.options.replication,
Collection: *worker.options.collection,
Ttl: *worker.options.ttl,
})
if err != nil {
fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err)
fmt.Printf("Failed to assign from %v: %v\n", worker.options.masters, err)
}
targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
@ -259,14 +365,12 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
uploadResult, err := operation.Upload(targetUrl,
fileName+"-"+strconv.FormatInt(i+1, 10),
io.LimitReader(f, chunkSize),
false, "application/octet-stream", nil, "")
false, "application/octet-stream", nil, assignResult.Auth)
if err != nil {
fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
return false
return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
}
if uploadResult.Error != "" {
fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
return false
return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
}
chunks = append(chunks, &filer_pb.FileChunk{
FileId: assignResult.Fid,
@ -278,39 +382,38 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
}
if err := withFilerClient(filerGrpcAddress, func(client filer_pb.SeaweedFilerClient) error {
if err := withFilerClient(ctx, worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.CreateEntryRequest{
Directory: urlFolder,
Directory: task.destinationUrlPath,
Entry: &filer_pb.Entry{
Name: fileName,
Attributes: &filer_pb.FuseAttributes{
Crtime: time.Now().Unix(),
Mtime: time.Now().Unix(),
Gid: uint32(os.Getgid()),
Uid: uint32(os.Getuid()),
FileSize: uint64(fi.Size()),
FileMode: uint32(fi.Mode()),
Gid: task.gid,
Uid: task.uid,
FileSize: uint64(task.fileSize),
FileMode: uint32(task.fileMode),
Mime: mimeType,
Replication: *copy.replication,
Collection: *copy.collection,
TtlSec: int32(util.ParseInt(*copy.ttl, 0)),
Replication: *worker.options.replication,
Collection: *worker.options.collection,
TtlSec: int32(util.ParseInt(*worker.options.ttl, 0)),
},
Chunks: chunks,
},
}
if _, err := client.CreateEntry(context.Background(), request); err != nil {
if _, err := client.CreateEntry(ctx, request); err != nil {
return fmt.Errorf("update fh: %v", err)
}
return nil
}); err != nil {
fmt.Printf("upload data %v to http://%s%s%s: %v\n", fileName, filerAddress, urlFolder, fileName, err)
return false
return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
}
fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerAddress, urlFolder, fileName)
fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName)
return true
return nil
}
func detectMimeType(f *os.File) string {
@ -329,15 +432,11 @@ func detectMimeType(f *os.File) string {
return mimeType
}
func withFilerClient(filerAddress string, fn func(filer_pb.SeaweedFilerClient) error) error {
grpcConnection, err := util.GrpcDial(filerAddress)
if err != nil {
return fmt.Errorf("fail to dial %s: %v", filerAddress, err)
}
defer grpcConnection.Close()
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
func withFilerClient(ctx context.Context, filerAddress string, grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {
return util.WithCachedGrpcClient(ctx, func(clientConn *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(clientConn)
return fn(client)
}, filerAddress, grpcDialOption)
}

187
weed/command/filer_export.go
View File

@ -1,187 +0,0 @@
package command
import (
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/notification"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/server"
"github.com/spf13/viper"
)
func init() {
cmdFilerExport.Run = runFilerExport // break init cycle
}
var cmdFilerExport = &Command{
UsageLine: "filer.export -sourceStore=mysql -targetStore=cassandra",
Short: "export meta data in filer store",
Long: `Iterate the file tree and export all metadata out
Both source and target store:
* should be a store name already specified in filer.toml
* do not need to be enabled state
If target store is empty, only the directory tree will be listed.
If target store is "notification", the list of entries will be sent to notification.
This is usually used to bootstrap filer replication to a remote system.
`,
}
var (
// filerExportOutputFile = cmdFilerExport.Flag.String("output", "", "the output file. If empty, only list out the directory tree")
filerExportSourceStore = cmdFilerExport.Flag.String("sourceStore", "", "the source store name in filer.toml, default to currently enabled store")
filerExportTargetStore = cmdFilerExport.Flag.String("targetStore", "", "the target store name in filer.toml, or \"notification\" to export all files to message queue")
dir = cmdFilerExport.Flag.String("dir", "/", "only process files under this directory")
dirListLimit = cmdFilerExport.Flag.Int("dirListLimit", 100000, "limit directory list size")
dryRun = cmdFilerExport.Flag.Bool("dryRun", false, "not actually moving data")
verboseFilerExport = cmdFilerExport.Flag.Bool("v", false, "verbose entry details")
)
type statistics struct {
directoryCount int
fileCount int
}
func runFilerExport(cmd *Command, args []string) bool {
weed_server.LoadConfiguration("filer", true)
config := viper.GetViper()
var sourceStore, targetStore filer2.FilerStore
for _, store := range filer2.Stores {
if store.GetName() == *filerExportSourceStore || *filerExportSourceStore == "" && config.GetBool(store.GetName()+".enabled") {
viperSub := config.Sub(store.GetName())
if err := store.Initialize(viperSub); err != nil {
glog.Fatalf("Failed to initialize source store for %s: %+v",
store.GetName(), err)
} else {
sourceStore = store
}
break
}
}
for _, store := range filer2.Stores {
if store.GetName() == *filerExportTargetStore {
viperSub := config.Sub(store.GetName())
if err := store.Initialize(viperSub); err != nil {
glog.Fatalf("Failed to initialize target store for %s: %+v",
store.GetName(), err)
} else {
targetStore = store
}
break
}
}
if sourceStore == nil {
glog.Errorf("Failed to find source store %s", *filerExportSourceStore)
println("existing data sources are:")
for _, store := range filer2.Stores {
println(" " + store.GetName())
}
return false
}
if targetStore == nil && *filerExportTargetStore != "" && *filerExportTargetStore != "notification" {
glog.Errorf("Failed to find target store %s", *filerExportTargetStore)
println("existing data sources are:")
for _, store := range filer2.Stores {
println(" " + store.GetName())
}
return false
}
stat := statistics{}
var fn func(level int, entry *filer2.Entry) error
if *filerExportTargetStore == "notification" {
weed_server.LoadConfiguration("notification", false)
v := viper.GetViper()
notification.LoadConfiguration(v.Sub("notification"))
fn = func(level int, entry *filer2.Entry) error {
printout(level, entry)
if *dryRun {
return nil
}
return notification.Queue.SendMessage(
string(entry.FullPath),
&filer_pb.EventNotification{
NewEntry: entry.ToProtoEntry(),
},
)
}
} else if targetStore == nil {
fn = printout
} else {
fn = func(level int, entry *filer2.Entry) error {
printout(level, entry)
if *dryRun {
return nil
}
return targetStore.InsertEntry(entry)
}
}
doTraverse(&stat, sourceStore, filer2.FullPath(*dir), 0, fn)
glog.Infof("processed %d directories, %d files", stat.directoryCount, stat.fileCount)
return true
}
func doTraverse(stat *statistics, filerStore filer2.FilerStore, parentPath filer2.FullPath, level int, fn func(level int, entry *filer2.Entry) error) {
limit := *dirListLimit
lastEntryName := ""
for {
entries, err := filerStore.ListDirectoryEntries(parentPath, lastEntryName, false, limit)
if err != nil {
break
}
for _, entry := range entries {
if fnErr := fn(level, entry); fnErr != nil {
glog.Errorf("failed to process entry: %s", entry.FullPath)
}
if entry.IsDirectory() {
stat.directoryCount++
doTraverse(stat, filerStore, entry.FullPath, level+1, fn)
} else {
stat.fileCount++
}
}
if len(entries) < limit {
break
}
}
}
func printout(level int, entry *filer2.Entry) error {
for i := 0; i < level; i++ {
if i == level-1 {
print("+-")
} else {
print("| ")
}
}
print(entry.FullPath.Name())
if *verboseFilerExport {
for _, chunk := range entry.Chunks {
print("[")
print(chunk.FileId)
print(",")
print(chunk.Offset)
print(",")
print(chunk.Size)
print(")")
}
}
println()
return nil
}

12
weed/command/filer_replication.go
View File

@ -1,6 +1,7 @@
package command
import (
"context"
"strings"
"github.com/chrislusf/seaweedfs/weed/glog"
@ -12,7 +13,7 @@ import (
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/gcssink"
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink"
"github.com/chrislusf/seaweedfs/weed/replication/sub"
"github.com/chrislusf/seaweedfs/weed/server"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/spf13/viper"
)
@ -28,15 +29,16 @@ var cmdFilerReplicate = &Command{
filer.replicate listens on filer notifications. If any file is updated, it will fetch the updated content,
and write to the other destination.
Run "weed scaffold -config replication" to generate a replication.toml file and customize the parameters.
Run "weed scaffold -config=replication" to generate a replication.toml file and customize the parameters.
`,
}
func runFilerReplicate(cmd *Command, args []string) bool {
weed_server.LoadConfiguration("replication", true)
weed_server.LoadConfiguration("notification", true)
util.LoadConfiguration("security", false)
util.LoadConfiguration("replication", true)
util.LoadConfiguration("notification", true)
config := viper.GetViper()
var notificationInput sub.NotificationInput
@ -115,7 +117,7 @@ func runFilerReplicate(cmd *Command, args []string) bool {
} else {
glog.V(1).Infof("modify: %s", key)
}
if err = replicator.Replicate(key, m); err != nil {
if err = replicator.Replicate(context.Background(), key, m); err != nil {
glog.Errorf("replicate %s: %+v", key, err)
} else {
glog.V(1).Infof("replicated %s", key)

13
weed/command/fix.go
View File

@ -7,6 +7,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/types"
)
@ -29,7 +30,7 @@ var (
)
type VolumeFileScanner4Fix struct {
version storage.Version
version needle.Version
nm *storage.NeedleMap
}
@ -42,14 +43,14 @@ func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
return false
}
func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *storage.Needle, offset int64) error {
func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64) error {
glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped())
if n.Size > 0 {
pe := scanner.nm.Put(n.Id, types.Offset(offset/types.NeedlePaddingSize), n.Size)
if n.Size > 0 && n.Size != types.TombstoneFileSize {
pe := scanner.nm.Put(n.Id, types.ToOffset(offset), n.Size)
glog.V(2).Infof("saved %d with error %v", n.Size, pe)
} else {
glog.V(2).Infof("skipping deleted file ...")
return scanner.nm.Delete(n.Id, types.Offset(offset/types.NeedlePaddingSize))
return scanner.nm.Delete(n.Id, types.ToOffset(offset))
}
return nil
}
@ -74,7 +75,7 @@ func runFix(cmd *Command, args []string) bool {
nm := storage.NewBtreeNeedleMap(indexFile)
defer nm.Close()
vid := storage.VolumeId(*fixVolumeId)
vid := needle.VolumeId(*fixVolumeId)
scanner := &VolumeFileScanner4Fix{
nm: nm,
}

133
weed/command/master.go
View File

@ -6,45 +6,70 @@ import (
"runtime"
"strconv"
"strings"
"time"
"github.com/chrislusf/raft/protobuf"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/server"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/gorilla/mux"
"github.com/spf13/viper"
"google.golang.org/grpc/reflection"
)
var (
m MasterOptions
)
type MasterOptions struct {
port *int
ip *string
ipBind *string
metaFolder *string
peers *string
volumeSizeLimitMB *uint
volumePreallocate *bool
pulseSeconds *int
defaultReplication *string
garbageThreshold *float64
whiteList *string
disableHttp *bool
metricsAddress *string
metricsIntervalSec *int
}
func init() {
cmdMaster.Run = runMaster // break init cycle
m.port = cmdMaster.Flag.Int("port", 9333, "http listen port")
m.ip = cmdMaster.Flag.String("ip", "localhost", "master <ip>|<server> address")
m.ipBind = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094")
m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
m.volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
m.pulseSeconds = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
m.defaultReplication = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
m.garbageThreshold = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
m.whiteList = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
m.disableHttp = cmdMaster.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
m.metricsAddress = cmdMaster.Flag.String("metrics.address", "", "Prometheus gateway address")
m.metricsIntervalSec = cmdMaster.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
}
var cmdMaster = &Command{
UsageLine: "master -port=9333",
Short: "start a master server",
Long: `start a master server to provide volume=>location mapping service
and sequence number of file ids
Long: `start a master server to provide volume=>location mapping service and sequence number of file ids
The configuration file "security.toml" is read from ".", "$HOME/.seaweedfs/", or "/etc/seaweedfs/", in that order.
The example security.toml configuration file can be generated by "weed scaffold -config=security"
`,
}
var (
mport = cmdMaster.Flag.Int("port", 9333, "http listen port")
mGrpcPort = cmdMaster.Flag.Int("port.grpc", 0, "grpc server listen port, default to http port + 10000")
masterIp = cmdMaster.Flag.String("ip", "localhost", "master <ip>|<server> address")
masterBindIp = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
masterPeers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094")
volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
mpulse = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
defaultReplicaPlacement = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
// mTimeout = cmdMaster.Flag.Int("idleTimeout", 30, "connection idle seconds")
mMaxCpu = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
garbageThreshold = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
masterWhiteListOption = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
masterSecureKey = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
masterCpuProfile = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")
masterMemProfile = cmdMaster.Flag.String("memprofile", "", "memory profile output file")
@ -52,30 +77,27 @@ var (
)
func runMaster(cmd *Command, args []string) bool {
if *mMaxCpu < 1 {
*mMaxCpu = runtime.NumCPU()
}
runtime.GOMAXPROCS(*mMaxCpu)
util.LoadConfiguration("security", false)
util.LoadConfiguration("master", false)
runtime.GOMAXPROCS(runtime.NumCPU())
util.SetupProfiling(*masterCpuProfile, *masterMemProfile)
if err := util.TestFolderWritable(*metaFolder); err != nil {
glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *metaFolder, err)
if err := util.TestFolderWritable(*m.metaFolder); err != nil {
glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *m.metaFolder, err)
}
if *masterWhiteListOption != "" {
masterWhiteList = strings.Split(*masterWhiteListOption, ",")
if *m.whiteList != "" {
masterWhiteList = strings.Split(*m.whiteList, ",")
}
if *volumeSizeLimitMB > 30*1000 {
if *m.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
}
r := mux.NewRouter()
ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
*volumeSizeLimitMB, *volumePreallocate,
*mpulse, *defaultReplicaPlacement, *garbageThreshold,
masterWhiteList, *masterSecureKey,
)
ms := weed_server.NewMasterServer(r, m.toMasterOption(masterWhiteList))
listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport)
listeningAddress := *m.ipBind + ":" + strconv.Itoa(*m.port)
glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress)
@ -85,28 +107,29 @@ func runMaster(cmd *Command, args []string) bool {
}
go func() {
time.Sleep(100 * time.Millisecond)
myMasterAddress, peers := checkPeers(*masterIp, *mport, *masterPeers)
raftServer := weed_server.NewRaftServer(r, peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
ms.SetRaftServer(raftServer)
}()
go func() {
// starting grpc server
grpcPort := *mGrpcPort
if grpcPort == 0 {
grpcPort = *mport + 10000
// start raftServer
myMasterAddress, peers := checkPeers(*m.ip, *m.port, *m.peers)
raftServer := weed_server.NewRaftServer(security.LoadClientTLS(viper.Sub("grpc"), "master"),
peers, myMasterAddress, *m.metaFolder, ms.Topo, *m.pulseSeconds)
if raftServer == nil {
glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717", *m.metaFolder)
}
grpcL, err := util.NewListener(*masterBindIp+":"+strconv.Itoa(grpcPort), 0)
ms.SetRaftServer(raftServer)
r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
// starting grpc server
grpcPort := *m.port + 10000
grpcL, err := util.NewListener(*m.ipBind+":"+strconv.Itoa(grpcPort), 0)
if err != nil {
glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
}
// Create your protocol servers.
grpcS := util.NewGrpcServer()
grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
master_pb.RegisterSeaweedServer(grpcS, ms)
protobuf.RegisterRaftServer(grpcS, raftServer)
reflection.Register(grpcS)
glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *masterBindIp, grpcPort)
glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *m.ipBind, grpcPort)
grpcS.Serve(grpcL)
}()
@ -142,3 +165,19 @@ func checkPeers(masterIp string, masterPort int, peers string) (masterAddress st
}
return
}
func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOption {
return &weed_server.MasterOption{
Port: *m.port,
MetaFolder: *m.metaFolder,
VolumeSizeLimitMB: *m.volumeSizeLimitMB,
VolumePreallocate: *m.volumePreallocate,
PulseSeconds: *m.pulseSeconds,
DefaultReplicaPlacement: *m.defaultReplication,
GarbageThreshold: *m.garbageThreshold,
WhiteList: whiteList,
DisableHttp: *m.disableHttp,
MetricsAddress: *m.metricsAddress,
MetricsIntervalSec: *m.metricsIntervalSec,
}
}

9
weed/command/mount.go
View File

@ -8,7 +8,6 @@ import (
type MountOptions struct {
filer *string
filerGrpcPort *int
filerMountRootPath *string
dir *string
dirListingLimit *int
@ -17,6 +16,7 @@ type MountOptions struct {
ttlSec *int
chunkSizeLimitMB *int
dataCenter *string
allowOthers *bool
}
var (
@ -28,7 +28,6 @@ var (
func init() {
cmdMount.Run = runMount // break init cycle
mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location")
mountOptions.filerGrpcPort = cmdMount.Flag.Int("filer.grpc.port", 0, "filer grpc server listen port, default to http port + 10000")
mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server")
mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory")
mountOptions.dirListingLimit = cmdMount.Flag.Int("dirListLimit", 100000, "limit directory listing size")
@ -37,6 +36,7 @@ func init() {
mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 4, "local write buffer size, also chunk large files")
mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
}
@ -59,7 +59,7 @@ var cmdMount = &Command{
`,
}
func parseFilerGrpcAddress(filer string, optionalGrpcPort int) (filerGrpcAddress string, err error) {
func parseFilerGrpcAddress(filer string) (filerGrpcAddress string, err error) {
hostnameAndPort := strings.Split(filer, ":")
if len(hostnameAndPort) != 2 {
return "", fmt.Errorf("The filer should have hostname:port format: %v", hostnameAndPort)
@ -71,9 +71,6 @@ func parseFilerGrpcAddress(filer string, optionalGrpcPort int) (filerGrpcAddress
}
filerGrpcPort := int(filerPort) + 10000
if optionalGrpcPort != 0 {
filerGrpcPort = optionalGrpcPort
}
return fmt.Sprintf("%s:%d", hostnameAndPort[0], filerGrpcPort), nil
}

89
weed/command/mount_std.go
View File

@ -6,11 +6,16 @@ import (
"fmt"
"os"
"os/user"
"path"
"runtime"
"strconv"
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/jacobsa/daemonize"
"github.com/spf13/viper"
"github.com/chrislusf/seaweedfs/weed/filesys"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/util"
@ -19,26 +24,52 @@ import (
)
func runMount(cmd *Command, args []string) bool {
util.SetupProfiling(*mountCpuProfile, *mountMemProfile)
return RunMount(
*mountOptions.filer,
*mountOptions.filerMountRootPath,
*mountOptions.dir,
*mountOptions.collection,
*mountOptions.replication,
*mountOptions.dataCenter,
*mountOptions.chunkSizeLimitMB,
*mountOptions.allowOthers,
*mountOptions.ttlSec,
*mountOptions.dirListingLimit,
)
}
func RunMount(filer, filerMountRootPath, dir, collection, replication, dataCenter string, chunkSizeLimitMB int,
allowOthers bool, ttlSec int, dirListingLimit int) bool {
util.LoadConfiguration("security", false)
fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
if *mountOptions.dir == "" {
if dir == "" {
fmt.Printf("Please specify the mount directory via \"-dir\"")
return false
}
if *mountOptions.chunkSizeLimitMB <= 0 {
if chunkSizeLimitMB <= 0 {
fmt.Printf("Please specify a reasonable buffer size.")
return false
}
fuse.Unmount(*mountOptions.dir)
fuse.Unmount(dir)
uid, gid := uint32(0), uint32(0)
// detect mount folder mode
mountMode := os.ModeDir | 0755
if fileInfo, err := os.Stat(*mountOptions.dir); err == nil {
fileInfo, err := os.Stat(dir)
if err == nil {
mountMode = os.ModeDir | fileInfo.Mode()
uid, gid = util.GetFileUidGid(fileInfo)
fmt.Printf("mount point owner uid=%d gid=%d mode=%s\n", uid, gid, fileInfo.Mode())
}
// detect current user
uid, gid := uint32(0), uint32(0)
if uid == 0 {
if u, err := user.Current(); err == nil {
if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
uid = uint32(parsedId)
@ -46,13 +77,14 @@ func runMount(cmd *Command, args []string) bool {
if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil {
gid = uint32(parsedId)
}
fmt.Printf("current uid=%d gid=%d\n", uid, gid)
}
}
util.SetupProfiling(*mountCpuProfile, *mountMemProfile)
mountName := path.Base(dir)
c, err := fuse.Mount(
*mountOptions.dir,
fuse.VolumeName("SeaweedFS"),
options := []fuse.MountOption{
fuse.VolumeName(mountName),
fuse.FSName("SeaweedFS"),
fuse.Subtype("SeaweedFS"),
fuse.NoAppleDouble(),
@ -61,56 +93,69 @@ func runMount(cmd *Command, args []string) bool {
fuse.AutoXattr(),
fuse.ExclCreate(),
fuse.DaemonTimeout("3600"),
fuse.AllowOther(),
fuse.AllowSUID(),
fuse.DefaultPermissions(),
fuse.MaxReadahead(1024 * 128),
fuse.AsyncRead(),
fuse.WritebackCache(),
)
fuse.AllowNonEmptyMount(),
}
if allowOthers {
options = append(options, fuse.AllowOther())
}
c, err := fuse.Mount(dir, options...)
if err != nil {
glog.Fatal(err)
daemonize.SignalOutcome(err)
return false
}
util.OnInterrupt(func() {
fuse.Unmount(*mountOptions.dir)
fuse.Unmount(dir)
c.Close()
})
filerGrpcAddress, err := parseFilerGrpcAddress(*mountOptions.filer, *mountOptions.filerGrpcPort)
filerGrpcAddress, err := parseFilerGrpcAddress(filer)
if err != nil {
glog.Fatal(err)
daemonize.SignalOutcome(err)
return false
}
mountRoot := *mountOptions.filerMountRootPath
mountRoot := filerMountRootPath
if mountRoot != "/" && strings.HasSuffix(mountRoot, "/") {
mountRoot = mountRoot[0 : len(mountRoot)-1]
}
daemonize.SignalOutcome(nil)
err = fs.Serve(c, filesys.NewSeaweedFileSystem(&filesys.Option{
FilerGrpcAddress: filerGrpcAddress,
GrpcDialOption: security.LoadClientTLS(viper.Sub("grpc"), "client"),
FilerMountRootPath: mountRoot,
Collection: *mountOptions.collection,
Replication: *mountOptions.replication,
TtlSec: int32(*mountOptions.ttlSec),
ChunkSizeLimit: int64(*mountOptions.chunkSizeLimitMB) * 1024 * 1024,
DataCenter: *mountOptions.dataCenter,
DirListingLimit: *mountOptions.dirListingLimit,
Collection: collection,
Replication: replication,
TtlSec: int32(ttlSec),
ChunkSizeLimit: int64(chunkSizeLimitMB) * 1024 * 1024,
DataCenter: dataCenter,
DirListingLimit: dirListingLimit,
EntryCacheTtl: 3 * time.Second,
MountUid: uid,
MountGid: gid,
MountMode: mountMode,
MountCtime: fileInfo.ModTime(),
MountMtime: time.Now(),
}))
if err != nil {
fuse.Unmount(*mountOptions.dir)
fuse.Unmount(dir)
}
// check if the mount process has an error to report
<-c.Ready
if err := c.MountError; err != nil {
glog.Fatal(err)
daemonize.SignalOutcome(err)
}
return true

47
weed/command/s3.go
View File

@ -4,7 +4,11 @@ import (
"net/http"
"time"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/spf13/viper"
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/s3api"
"github.com/chrislusf/seaweedfs/weed/util"
@ -12,12 +16,11 @@ import (
)
var (
s3options S3Options
s3StandaloneOptions S3Options
)
type S3Options struct {
filer *string
filerGrpcPort *int
filerBucketsPath *string
port *int
domainName *string
@ -27,13 +30,12 @@ type S3Options struct {
func init() {
cmdS3.Run = runS3 // break init cycle
s3options.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
s3options.filerGrpcPort = cmdS3.Flag.Int("filer.grpcPort", 0, "filer server grpc port, default to filer http port plus 10000")
s3options.filerBucketsPath = cmdS3.Flag.String("filer.dir.buckets", "/buckets", "folder on filer to store all buckets")
s3options.port = cmdS3.Flag.Int("port", 8333, "s3options server http listen port")
s3options.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name, {bucket}.{domainName}")
s3options.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
s3options.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
s3StandaloneOptions.filerBucketsPath = cmdS3.Flag.String("filer.dir.buckets", "/buckets", "folder on filer to store all buckets")
s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name, {bucket}.{domainName}")
s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
}
var cmdS3 = &Command{
@ -46,7 +48,15 @@ var cmdS3 = &Command{
func runS3(cmd *Command, args []string) bool {
filerGrpcAddress, err := parseFilerGrpcAddress(*s3options.filer, *s3options.filerGrpcPort)
util.LoadConfiguration("security", false)
return s3StandaloneOptions.startS3Server()
}
func (s3opt *S3Options) startS3Server() bool {
filerGrpcAddress, err := parseFilerGrpcAddress(*s3opt.filer)
if err != nil {
glog.Fatal(err)
return false
@ -55,10 +65,11 @@ func runS3(cmd *Command, args []string) bool {
router := mux.NewRouter().SkipClean(true)
_, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
Filer: *s3options.filer,
Filer: *s3opt.filer,
FilerGrpcAddress: filerGrpcAddress,
DomainName: *s3options.domainName,
BucketsPath: *s3options.filerBucketsPath,
DomainName: *s3opt.domainName,
BucketsPath: *s3opt.filerBucketsPath,
GrpcDialOption: security.LoadClientTLS(viper.Sub("grpc"), "client"),
})
if s3ApiServer_err != nil {
glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
@ -66,22 +77,22 @@ func runS3(cmd *Command, args []string) bool {
httpS := &http.Server{Handler: router}
listenAddress := fmt.Sprintf(":%d", *s3options.port)
listenAddress := fmt.Sprintf(":%d", *s3opt.port)
s3ApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
if err != nil {
glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
}
if *s3options.tlsPrivateKey != "" {
if err = httpS.ServeTLS(s3ApiListener, *s3options.tlsCertificate, *s3options.tlsPrivateKey); err != nil {
if *s3opt.tlsPrivateKey != "" {
glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.VERSION, *s3opt.port)
if err = httpS.ServeTLS(s3ApiListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
glog.Fatalf("S3 API Server Fail to serve: %v", err)
}
glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.VERSION, *s3options.port)
} else {
glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.VERSION, *s3opt.port)
if err = httpS.Serve(s3ApiListener); err != nil {
glog.Fatalf("S3 API Server Fail to serve: %v", err)
}
glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.VERSION, *s3options.port)
}
return true

109
weed/command/scaffold.go
View File

@ -10,7 +10,7 @@ func init() {
}
var cmdScaffold = &Command{
UsageLine: "scaffold [filer]",
UsageLine: "scaffold -config=[filer|notification|replication|security|master]",
Short: "generate basic configuration files",
Long: `Generate filer.toml with all possible configurations for you to customize.
@ -19,7 +19,7 @@ var cmdScaffold = &Command{
var (
outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory")
config = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication] the configuration file to generate")
config = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security|master] the configuration file to generate")
)
func runScaffold(cmd *Command, args []string) bool {
@ -32,6 +32,10 @@ func runScaffold(cmd *Command, args []string) bool {
content = NOTIFICATION_TOML_EXAMPLE
case "replication":
content = REPLICATION_TOML_EXAMPLE
case "security":
content = SECURITY_TOML_EXAMPLE
case "master":
content = MASTER_TOML_EXAMPLE
}
if content == "" {
println("need a valid -config option")
@ -61,6 +65,12 @@ enabled = false
[leveldb]
# local on disk, mostly for simple single-machine setup, fairly scalable
enabled = false
dir = "." # directory to store level db files
[leveldb2]
# local on disk, mostly for simple single-machine setup, fairly scalable
# faster than previous leveldb, recommended.
enabled = true
dir = "." # directory to store level db files
@ -72,10 +82,11 @@ dir = "." # directory to store level db files
# CREATE TABLE IF NOT EXISTS filemeta (
# dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
# name VARCHAR(1000) COMMENT 'directory or file name',
# directory VARCHAR(4096) COMMENT 'full path to parent directory',
# meta BLOB,
# directory TEXT COMMENT 'full path to parent directory',
# meta LONGBLOB,
# PRIMARY KEY (dirhash, name)
# ) DEFAULT CHARSET=utf8;
enabled = false
hostname = "localhost"
port = 3306
@ -88,8 +99,8 @@ connection_max_open = 100
[postgres]
# CREATE TABLE IF NOT EXISTS filemeta (
# dirhash BIGINT,
# name VARCHAR(1000),
# directory VARCHAR(4096),
# name VARCHAR(65535),
# directory VARCHAR(65535),
# meta bytea,
# PRIMARY KEY (dirhash, name)
# );
@ -132,6 +143,7 @@ addresses = [
"localhost:30005",
"localhost:30006",
]
password = ""
`
@ -178,6 +190,17 @@ google_application_credentials = "/path/to/x.json" # path to json credential fil
project_id = "" # an existing project id
topic = "seaweedfs_filer_topic" # a topic, auto created if does not exists
[notification.gocdk_pub_sub]
# The Go Cloud Development Kit (https://gocloud.dev).
# PubSub API (https://godoc.org/gocloud.dev/pubsub).
# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ.
enabled = false
# This URL will Dial the RabbitMQ server at the URL in the environment
# variable RABBIT_SERVER_URL and open the exchange "myexchange".
# The exchange must have already been created by some other means, like
# the RabbitMQ management plugin.
topic_url = "rabbit://myexchange"
sub_url = "rabbit://myqueue"
`
REPLICATION_TOML_EXAMPLE = `
@ -239,5 +262,79 @@ b2_master_application_key = ""
bucket = "mybucket" # an existing bucket
directory = "/" # destination directory
`
SECURITY_TOML_EXAMPLE = `
# Put this file to one of the location, with descending priority
# ./security.toml
# $HOME/.seaweedfs/security.toml
# /etc/seaweedfs/security.toml
# this file is read by master, volume server, and filer
# the jwt signing key is read by master and volume server.
# a jwt defaults to expire after 10 seconds.
[jwt.signing]
key = ""
expires_after_seconds = 10 # seconds
# jwt for read is only supported with master+volume setup. Filer does not support this mode.
[jwt.signing.read]
key = ""
expires_after_seconds = 10 # seconds
# all grpc tls authentications are mutual
# the values for the following ca, cert, and key are paths to the PERM files.
# the host name is not checked, so the PERM files can be shared.
[grpc]
ca = ""
[grpc.volume]
cert = ""
key = ""
[grpc.master]
cert = ""
key = ""
[grpc.filer]
cert = ""
key = ""
# use this for any place needs a grpc client
# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
[grpc.client]
cert = ""
key = ""
# volume server https options
# Note: work in progress!
# this does not work with other clients, e.g., "weed filer|mount" etc, yet.
[https.client]
enabled = true
[https.volume]
cert = ""
key = ""
`
MASTER_TOML_EXAMPLE = `
# Put this file to one of the location, with descending priority
# ./master.toml
# $HOME/.seaweedfs/master.toml
# /etc/seaweedfs/master.toml
# this file is read by master
[master.maintenance]
# periodically run these scripts are the same as running them from 'weed shell'
scripts = """
ec.encode -fullPercent=95 -quietFor=1h
ec.rebuild -force
ec.balance -force
volume.balance -force
"""
sleep_minutes = 17 # sleep minutes between each script execution
`
)

144
weed/command/server.go
View File

@ -1,6 +1,7 @@
package command
import (
"fmt"
"net/http"
"os"
"runtime"
@ -10,6 +11,10 @@ import (
"sync"
"time"
"github.com/chrislusf/raft/protobuf"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/spf13/viper"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/server"
@ -25,7 +30,9 @@ type ServerOptions struct {
var (
serverOptions ServerOptions
masterOptions MasterOptions
filerOptions FilerOptions
s3Options S3Options
)
func init() {
@ -34,17 +41,16 @@ func init() {
var cmdServer = &Command{
UsageLine: "server -port=8080 -dir=/tmp -volume.max=5 -ip=server_name",
Short: "start a server, including volume server, and automatically elect a master server",
Short: "start a master server, a volume server, and optionally a filer and a S3 gateway",
Long: `start both a volume server to provide storage spaces
and a master server to provide volume=>location mapping service and sequence number of file ids
This is provided as a convenient way to start both volume server and master server.
The servers are exactly the same as starting them separately.
The servers acts exactly the same as starting them separately.
So other volume servers can connect to this master server also.
So other volume servers can use this embedded master server also.
Optionally, one filer server can be started. Logically, filer servers should not be in a cluster.
They run with meta data on disk, not shared. So each filer server is different.
Optionally, a filer server can be started.
Also optionally, a S3 gateway can be started.
`,
}
@ -52,33 +58,35 @@ var cmdServer = &Command{
var (
serverIp = cmdServer.Flag.String("ip", "localhost", "ip or server name")
serverBindIp = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
serverMaxCpu = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
serverTimeout = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name")
serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
serverPeers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
serverSecureKey = cmdServer.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
serverGarbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
masterPort = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
masterGrpcPort = cmdServer.Flag.Int("master.port.grpc", 0, "master grpc server listen port, default to http port + 10000")
masterMetaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
masterVolumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
masterVolumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.")
serverDisableHttp = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
volumeDataFolders = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...")
pulseSeconds = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer")
isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
serverWhiteList []string
)
func init() {
serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "000", "Default replication type if not specified.")
masterOptions.garbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
filerOptions.grpcPort = cmdServer.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to http port + 10000")
filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
@ -88,15 +96,25 @@ func init() {
serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
s3Options.filerBucketsPath = cmdServer.Flag.String("s3.filer.dir.buckets", "/buckets", "folder on filer to store all buckets")
s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}")
s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
}
func runServer(cmd *Command, args []string) bool {
filerOptions.secretKey = serverSecureKey
util.LoadConfiguration("security", false)
util.LoadConfiguration("master", false)
if *serverOptions.cpuprofile != "" {
f, err := os.Create(*serverOptions.cpuprofile)
if err != nil {
@ -110,41 +128,53 @@ func runServer(cmd *Command, args []string) bool {
*isStartingFiler = true
}
master := *serverIp + ":" + strconv.Itoa(*masterPort)
filerOptions.ip = serverIp
if *isStartingS3 {
*isStartingFiler = true
}
master := *serverIp + ":" + strconv.Itoa(*masterOptions.port)
masterOptions.ip = serverIp
masterOptions.ipBind = serverBindIp
filerOptions.masters = &master
filerOptions.ip = serverBindIp
serverOptions.v.ip = serverIp
serverOptions.v.bindIp = serverBindIp
serverOptions.v.masters = &master
serverOptions.v.idleConnectionTimeout = serverTimeout
serverOptions.v.maxCpu = serverMaxCpu
serverOptions.v.dataCenter = serverDataCenter
serverOptions.v.rack = serverRack
serverOptions.v.pulseSeconds = pulseSeconds
masterOptions.pulseSeconds = pulseSeconds
masterOptions.whiteList = serverWhiteListOption
filerOptions.dataCenter = serverDataCenter
filerOptions.disableHttp = serverDisableHttp
masterOptions.disableHttp = serverDisableHttp
filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port)
s3Options.filer = &filerAddress
if *filerOptions.defaultReplicaPlacement == "" {
*filerOptions.defaultReplicaPlacement = *masterDefaultReplicaPlacement
*filerOptions.defaultReplicaPlacement = *masterOptions.defaultReplication
}
if *serverMaxCpu < 1 {
*serverMaxCpu = runtime.NumCPU()
}
runtime.GOMAXPROCS(*serverMaxCpu)
runtime.GOMAXPROCS(runtime.NumCPU())
folders := strings.Split(*volumeDataFolders, ",")
if *masterVolumeSizeLimitMB > 30*1000 {
if *masterOptions.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
}
if *masterMetaFolder == "" {
*masterMetaFolder = folders[0]
if *masterOptions.metaFolder == "" {
*masterOptions.metaFolder = folders[0]
}
if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err)
if err := util.TestFolderWritable(*masterOptions.metaFolder); err != nil {
glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterOptions.metaFolder, err)
}
filerOptions.defaultLevelDbDirectory = masterMetaFolder
filerOptions.defaultLevelDbDirectory = masterOptions.metaFolder
if *serverWhiteListOption != "" {
serverWhiteList = strings.Split(*serverWhiteListOption, ",")
@ -159,55 +189,55 @@ func runServer(cmd *Command, args []string) bool {
}()
}
var raftWaitForMaster sync.WaitGroup
if *isStartingS3 {
go func() {
time.Sleep(2 * time.Second)
s3Options.startS3Server()
}()
}
var volumeWait sync.WaitGroup
raftWaitForMaster.Add(1)
volumeWait.Add(1)
go func() {
r := mux.NewRouter()
ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
*masterVolumeSizeLimitMB, *masterVolumePreallocate,
*pulseSeconds, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
serverWhiteList, *serverSecureKey,
)
ms := weed_server.NewMasterServer(r, masterOptions.toMasterOption(serverWhiteList))
glog.V(0).Infof("Start Seaweed Master %s at %s:%d", util.VERSION, *serverIp, *masterPort)
masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), 0)
glog.V(0).Infof("Start Seaweed Master %s at %s:%d", util.VERSION, *serverIp, *masterOptions.port)
masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterOptions.port), 0)
if e != nil {
glog.Fatalf("Master startup error: %v", e)
}
go func() {
// start raftServer
myMasterAddress, peers := checkPeers(*serverIp, *masterOptions.port, *masterOptions.peers)
raftServer := weed_server.NewRaftServer(security.LoadClientTLS(viper.Sub("grpc"), "master"),
peers, myMasterAddress, *masterOptions.metaFolder, ms.Topo, *masterOptions.pulseSeconds)
ms.SetRaftServer(raftServer)
r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
// starting grpc server
grpcPort := *masterGrpcPort
if grpcPort == 0 {
grpcPort = *masterPort + 10000
}
grpcL, err := util.NewListener(*serverIp+":"+strconv.Itoa(grpcPort), 0)
grpcPort := *masterOptions.port + 10000
grpcL, err := util.NewListener(*serverBindIp+":"+strconv.Itoa(grpcPort), 0)
if err != nil {
glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
}
// Create your protocol servers.
grpcS := util.NewGrpcServer()
glog.V(1).Infof("grpc config %+v", viper.Sub("grpc"))
grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
master_pb.RegisterSeaweedServer(grpcS, ms)
protobuf.RegisterRaftServer(grpcS, raftServer)
reflection.Register(grpcS)
glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *serverIp, grpcPort)
grpcS.Serve(grpcL)
}()
go func() {
raftWaitForMaster.Wait()
time.Sleep(100 * time.Millisecond)
myAddress, peers := checkPeers(*serverIp, *masterPort, *serverPeers)
raftServer := weed_server.NewRaftServer(r, peers, myAddress, *masterMetaFolder, ms.Topo, *pulseSeconds)
ms.SetRaftServer(raftServer)
volumeWait.Done()
}()
raftWaitForMaster.Done()
// start http server
httpS := &http.Server{Handler: r}

61
weed/command/shell.go
View File

@ -1,21 +1,25 @@
package command
import (
"bufio"
"fmt"
"os"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/shell"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/spf13/viper"
)
"github.com/chrislusf/seaweedfs/weed/glog"
var (
shellOptions shell.ShellOptions
)
func init() {
cmdShell.Run = runShell // break init cycle
shellOptions.Masters = cmdShell.Flag.String("master", "localhost:9333", "comma-separated master servers")
}
var cmdShell = &Command{
UsageLine: "shell",
Short: "run interactive commands, now just echo",
Long: `run interactive commands.
Short: "run interactive administrative commands",
Long: `run interactive administrative commands.
`,
}
@ -23,39 +27,16 @@ var cmdShell = &Command{
var ()
func runShell(command *Command, args []string) bool {
r := bufio.NewReader(os.Stdin)
o := bufio.NewWriter(os.Stdout)
e := bufio.NewWriter(os.Stderr)
prompt := func() {
var err error
if _, err = o.WriteString("> "); err != nil {
glog.V(0).Infoln("error writing to stdout:", err)
}
if err = o.Flush(); err != nil {
glog.V(0).Infoln("error flushing stdout:", err)
}
}
readLine := func() string {
ret, err := r.ReadString('\n')
if err != nil {
fmt.Fprint(e, err)
os.Exit(1)
}
return ret
}
execCmd := func(cmd string) int {
if cmd != "" {
if _, err := o.WriteString(cmd); err != nil {
glog.V(0).Infoln("error writing to stdout:", err)
}
}
return 0
}
cmd := ""
for {
prompt()
cmd = readLine()
execCmd(cmd)
}
util.LoadConfiguration("security", false)
shellOptions.GrpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
shellOptions.FilerHost = "localhost"
shellOptions.FilerPort = 8888
shellOptions.Directory = "/"
shell.RunShell(shellOptions)
return true
}

24
weed/command/upload.go
View File

@ -6,8 +6,11 @@ import (
"os"
"path/filepath"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/spf13/viper"
"github.com/chrislusf/seaweedfs/weed/operation"
)
var (
@ -23,7 +26,6 @@ type UploadOptions struct {
dataCenter *string
ttl *string
maxMB *int
secretKey *string
}
func init() {
@ -36,8 +38,7 @@ func init() {
upload.collection = cmdUpload.Flag.String("collection", "", "optional collection name")
upload.dataCenter = cmdUpload.Flag.String("dataCenter", "", "optional data center name")
upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
upload.maxMB = cmdUpload.Flag.Int("maxMB", 0, "split files larger than the limit")
upload.secretKey = cmdUpload.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
upload.maxMB = cmdUpload.Flag.Int("maxMB", 32, "split files larger than the limit")
}
var cmdUpload = &Command{
@ -53,14 +54,17 @@ var cmdUpload = &Command{
All files under the folder and subfolders will be uploaded, each with its own file key.
Optional parameter "-include" allows you to specify the file name patterns.
If "maxMB" is set to a positive number, files larger than it would be split into chunks and uploaded separatedly.
If "maxMB" is set to a positive number, files larger than it would be split into chunks and uploaded separately.
The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned.
`,
}
func runUpload(cmd *Command, args []string) bool {
secret := security.Secret(*upload.secretKey)
util.LoadConfiguration("security", false)
grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
if len(args) == 0 {
if *upload.dir == "" {
return false
@ -77,9 +81,9 @@ func runUpload(cmd *Command, args []string) bool {
if e != nil {
return e
}
results, e := operation.SubmitFiles(*upload.master, parts,
results, e := operation.SubmitFiles(*upload.master, grpcDialOption, parts,
*upload.replication, *upload.collection, *upload.dataCenter,
*upload.ttl, *upload.maxMB, secret)
*upload.ttl, *upload.maxMB)
bytes, _ := json.Marshal(results)
fmt.Println(string(bytes))
if e != nil {
@ -96,9 +100,9 @@ func runUpload(cmd *Command, args []string) bool {
if e != nil {
fmt.Println(e.Error())
}
results, _ := operation.SubmitFiles(*upload.master, parts,
results, _ := operation.SubmitFiles(*upload.master, grpcDialOption, parts,
*upload.replication, *upload.collection, *upload.dataCenter,
*upload.ttl, *upload.maxMB, secret)
*upload.ttl, *upload.maxMB)
bytes, _ := json.Marshal(results)
fmt.Println(string(bytes))
}

35
weed/command/volume.go
View File

@ -9,6 +9,9 @@ import (
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/spf13/viper"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/server"
@ -32,7 +35,6 @@ type VolumeServerOptions struct {
masters *string
pulseSeconds *int
idleConnectionTimeout *int
maxCpu *int
dataCenter *string
rack *string
whiteList []string
@ -41,6 +43,7 @@ type VolumeServerOptions struct {
readRedirect *bool
cpuProfile *string
memProfile *string
compactionMBPerSecond *int
}
func init() {
@ -53,14 +56,14 @@ func init() {
v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds")
v.maxCpu = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
}
var cmdVolume = &Command{
@ -78,10 +81,10 @@ var (
)
func runVolume(cmd *Command, args []string) bool {
if *v.maxCpu < 1 {
*v.maxCpu = runtime.NumCPU()
}
runtime.GOMAXPROCS(*v.maxCpu)
util.LoadConfiguration("security", false)
runtime.GOMAXPROCS(runtime.NumCPU())
util.SetupProfiling(*v.cpuProfile, *v.memProfile)
v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption)
@ -137,10 +140,10 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
switch *v.indexType {
case "leveldb":
volumeNeedleMapKind = storage.NeedleMapLevelDb
case "boltdb":
volumeNeedleMapKind = storage.NeedleMapBoltDb
case "btree":
volumeNeedleMapKind = storage.NeedleMapBtree
case "leveldbMedium":
volumeNeedleMapKind = storage.NeedleMapLevelDbMedium
case "leveldbLarge":
volumeNeedleMapKind = storage.NeedleMapLevelDbLarge
}
masters := *v.masters
@ -152,6 +155,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
strings.Split(masters, ","), *v.pulseSeconds, *v.dataCenter, *v.rack,
v.whiteList,
*v.fixJpgOrientation, *v.readRedirect,
*v.compactionMBPerSecond,
)
listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)
@ -185,13 +189,20 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
if err != nil {
glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
}
grpcS := util.NewGrpcServer()
grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "volume"))
volume_server_pb.RegisterVolumeServerServer(grpcS, volumeServer)
reflection.Register(grpcS)
go grpcS.Serve(grpcL)
if viper.GetString("https.volume.key") != "" {
if e := http.ServeTLS(listener, volumeMux,
viper.GetString("https.volume.cert"), viper.GetString("https.volume.key")); e != nil {
glog.Fatalf("Volume server fail to serve: %v", e)
}
} else {
if e := http.Serve(listener, volumeMux); e != nil {
glog.Fatalf("Volume server fail to serve: %v", e)
}
}
}

109
weed/command/webdav.go Normal file
View File

@ -0,0 +1,109 @@
package command
import (
"fmt"
"net/http"
"os/user"
"strconv"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/server"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/spf13/viper"
)
var (
webDavStandaloneOptions WebDavOption
)
type WebDavOption struct {
filer *string
port *int
collection *string
tlsPrivateKey *string
tlsCertificate *string
}
func init() {
cmdWebDav.Run = runWebDav // break init cycle
webDavStandaloneOptions.filer = cmdWebDav.Flag.String("filer", "localhost:8888", "filer server address")
webDavStandaloneOptions.port = cmdWebDav.Flag.Int("port", 7333, "webdav server http listen port")
webDavStandaloneOptions.collection = cmdWebDav.Flag.String("collection", "", "collection to create the files")
webDavStandaloneOptions.tlsPrivateKey = cmdWebDav.Flag.String("key.file", "", "path to the TLS private key file")
webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file")
}
var cmdWebDav = &Command{
UsageLine: "webdav -port=7333 -filer=<ip:port>",
Short: "<unstable> start a webdav server that is backed by a filer",
Long: `start a webdav server that is backed by a filer.
`,
}
func runWebDav(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false)
glog.V(0).Infof("Starting Seaweed WebDav Server %s at https port %d", util.VERSION, *webDavStandaloneOptions.port)
return webDavStandaloneOptions.startWebDav()
}
func (wo *WebDavOption) startWebDav() bool {
filerGrpcAddress, err := parseFilerGrpcAddress(*wo.filer)
if err != nil {
glog.Fatal(err)
return false
}
// detect current user
uid, gid := uint32(0), uint32(0)
if u, err := user.Current(); err == nil {
if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
uid = uint32(parsedId)
}
if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil {
gid = uint32(parsedId)
}
}
ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{
Filer: *wo.filer,
FilerGrpcAddress: filerGrpcAddress,
GrpcDialOption: security.LoadClientTLS(viper.Sub("grpc"), "client"),
Collection: *wo.collection,
Uid: uid,
Gid: gid,
})
if webdavServer_err != nil {
glog.Fatalf("WebDav Server startup error: %v", webdavServer_err)
}
httpS := &http.Server{Handler: ws.Handler}
listenAddress := fmt.Sprintf(":%d", *wo.port)
webDavListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
if err != nil {
glog.Fatalf("WebDav Server listener on %s error: %v", listenAddress, err)
}
if *wo.tlsPrivateKey != "" {
glog.V(0).Infof("Start Seaweed WebDav Server %s at https port %d", util.VERSION, *wo.port)
if err = httpS.ServeTLS(webDavListener, *wo.tlsCertificate, *wo.tlsPrivateKey); err != nil {
glog.Fatalf("WebDav Server Fail to serve: %v", err)
}
} else {
glog.V(0).Infof("Start Seaweed WebDav Server %s at http port %d", util.VERSION, *wo.port)
if err = httpS.Serve(webDavListener); err != nil {
glog.Fatalf("WebDav Server Fail to serve: %v", err)
}
}
return true
}

84
weed/command/weedfuse/README.md Normal file
View File

@ -0,0 +1,84 @@
Mount the SeaweedFS via FUSE
# Mount by fstab
```
$ # on linux
$ sudo apt-get install fuse
$ sudo echo 'user_allow_other' >> /etc/fuse.conf
$ sudo mv weedfuse /sbin/mount.weedfuse
$ # on Mac
$ sudo mv weedfuse /sbin/mount_weedfuse
```
On both OS X and Linux, you can add one of the entries to your /etc/fstab file like the following:
```
# mount the whole SeaweedFS
localhost:8888/ /home/some/mount/folder weedfuse
# mount the SeaweedFS sub folder
localhost:8888/sub/dir /home/some/mount/folder weedfuse
# mount the SeaweedFS sub folder with some options
localhost:8888/sub/dir /home/some/mount/folder weedfuse user
```
To verify it can work, try this command
```
$ sudo mount -av
...
/home/some/mount/folder : successfully mounted
```
If you see `successfully mounted`, try to access the mounted folder and verify everything works.
To debug, run these:
```
$ weedfuse -foreground localhost:8888/ /home/some/mount/folder
```
To unmount the folder:
```
$ sudo umount /home/some/mount/folder
```
<!-- not working yet!
# Mount by autofs
AutoFS can mount a folder if accessed.
```
# install autofs
$ sudo apt-get install autofs
```
Here is an example on how to mount a folder for all users under `/home` directory.
Assuming there exists corresponding folders under `/home` on both local and SeaweedFS.
Edit `/etc/auto.master` and `/etc/auto.weedfuse` file with these content
```
$ cat /etc/auto.master
/home /etc/auto.weedfuse
$ cat /etc/auto.weedfuse
# map /home/<user> to localhost:8888/home/<user>
* -fstype=weedfuse,rw,allow_other,foreground :localhost\:8888/home/&
```
-->

109
weed/command/weedfuse/weedfuse.go Normal file
View File

@ -0,0 +1,109 @@
package main
import (
"flag"
"fmt"
"os"
"strings"
"github.com/chrislusf/seaweedfs/weed/command"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/jacobsa/daemonize"
"github.com/kardianos/osext"
)
var (
fuseCommand = flag.NewFlagSet("weedfuse", flag.ContinueOnError)
options = fuseCommand.String("o", "", "comma separated options rw,uid=xxx,gid=xxx")
isForeground = fuseCommand.Bool("foreground", false, "starts as a daemon")
)
func main() {
err := fuseCommand.Parse(os.Args[1:])
if err != nil {
glog.Fatal(err)
}
fmt.Printf("options: %v\n", *options)
// seems this value is always empty, need to parse it differently
optionsString := *options
prev := ""
for i, arg := range os.Args {
fmt.Printf("args[%d]: %v\n", i, arg)
if prev == "-o" {
optionsString = arg
}
prev = arg
}
device := fuseCommand.Arg(0)
mountPoint := fuseCommand.Arg(1)
fmt.Printf("source: %v\n", device)
fmt.Printf("target: %v\n", mountPoint)
nouser := true
for _, option := range strings.Split(optionsString, ",") {
fmt.Printf("option: %v\n", option)
switch option {
case "user":
nouser = false
}
}
maybeSetupPath()
if !*isForeground {
startAsDaemon()
return
}
parts := strings.SplitN(device, "/", 2)
filer, filerPath := parts[0], parts[1]
command.RunMount(
filer, "/"+filerPath, mountPoint, "", "000", "",
4, !nouser, 0, 1000000)
}
func maybeSetupPath() {
// sudo mount -av may not include PATH in some linux, e.g., Ubuntu
hasPathEnv := false
for _, e := range os.Environ() {
if strings.HasPrefix(e, "PATH=") {
hasPathEnv = true
}
fmt.Println(e)
}
if !hasPathEnv {
os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
}
}
func startAsDaemon() {
// adapted from gcsfuse
// Find the executable.
var path string
path, err := osext.Executable()
if err != nil {
glog.Fatalf("osext.Executable: %v", err)
}
// Set up arguments. Be sure to use foreground mode.
args := append([]string{"-foreground"}, os.Args[1:]...)
// Pass along PATH so that the daemon can find fusermount on Linux.
env := []string{
fmt.Sprintf("PATH=%s", os.Getenv("PATH")),
}
err = daemonize.Run(path, args, env, os.Stdout)
if err != nil {
glog.Fatalf("daemonize.Run: %v", err)
}
}

58
weed/filer2/abstract_sql/abstract_sql_store.go
View File

@ -1,6 +1,7 @@
package abstract_sql
import (
"context"
"database/sql"
"fmt"
@ -18,7 +19,44 @@ type AbstractSqlStore struct {
SqlListInclusive string
}
func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) {
type TxOrDB interface {
ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row
QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
}
func (store *AbstractSqlStore) BeginTransaction(ctx context.Context) (context.Context, error) {
tx, err := store.DB.BeginTx(ctx, &sql.TxOptions{
Isolation: sql.LevelReadCommitted,
ReadOnly: false,
})
if err != nil {
return ctx, err
}
return context.WithValue(ctx, "tx", tx), nil
}
func (store *AbstractSqlStore) CommitTransaction(ctx context.Context) error {
if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
return tx.Commit()
}
return nil
}
func (store *AbstractSqlStore) RollbackTransaction(ctx context.Context) error {
if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
return tx.Rollback()
}
return nil
}
func (store *AbstractSqlStore) getTxOrDB(ctx context.Context) TxOrDB {
if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
return tx
}
return store.DB
}
func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
dir, name := entry.FullPath.DirAndName()
meta, err := entry.EncodeAttributesAndChunks()
@ -26,7 +64,7 @@ func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) {
return fmt.Errorf("encode %s: %s", entry.FullPath, err)
}
res, err := store.DB.Exec(store.SqlInsert, hashToLong(dir), name, dir, meta)
res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, hashToLong(dir), name, dir, meta)
if err != nil {
return fmt.Errorf("insert %s: %s", entry.FullPath, err)
}
@ -38,7 +76,7 @@ func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) {
return nil
}
func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) {
func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
dir, name := entry.FullPath.DirAndName()
meta, err := entry.EncodeAttributesAndChunks()
@ -46,7 +84,7 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) {
return fmt.Errorf("encode %s: %s", entry.FullPath, err)
}
res, err := store.DB.Exec(store.SqlUpdate, meta, hashToLong(dir), name, dir)
res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, meta, hashToLong(dir), name, dir)
if err != nil {
return fmt.Errorf("update %s: %s", entry.FullPath, err)
}
@ -58,10 +96,10 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) {
return nil
}
func (store *AbstractSqlStore) FindEntry(fullpath filer2.FullPath) (*filer2.Entry, error) {
func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (*filer2.Entry, error) {
dir, name := fullpath.DirAndName()
row := store.DB.QueryRow(store.SqlFind, hashToLong(dir), name, dir)
row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, hashToLong(dir), name, dir)
var data []byte
if err := row.Scan(&data); err != nil {
return nil, filer2.ErrNotFound
@ -77,11 +115,11 @@ func (store *AbstractSqlStore) FindEntry(fullpath filer2.FullPath) (*filer2.Entr
return entry, nil
}
func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) error {
func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) error {
dir, name := fullpath.DirAndName()
res, err := store.DB.Exec(store.SqlDelete, hashToLong(dir), name, dir)
res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDelete, hashToLong(dir), name, dir)
if err != nil {
return fmt.Errorf("delete %s: %s", fullpath, err)
}
@ -94,14 +132,14 @@ func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) error {
return nil
}
func (store *AbstractSqlStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
sqlText := store.SqlListExclusive
if inclusive {
sqlText = store.SqlListInclusive
}
rows, err := store.DB.Query(sqlText, hashToLong(string(fullpath)), startFileName, string(fullpath), limit)
rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, hashToLong(string(fullpath)), startFileName, string(fullpath), limit)
if err != nil {
return nil, fmt.Errorf("list %s : %v", fullpath, err)
}

25
weed/filer2/cassandra/cassandra_store.go
View File

@ -1,6 +1,7 @@
package cassandra
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
@ -39,7 +40,17 @@ func (store *CassandraStore) initialize(keyspace string, hosts []string) (err er
return
}
func (store *CassandraStore) InsertEntry(entry *filer2.Entry) (err error) {
func (store *CassandraStore) BeginTransaction(ctx context.Context) (context.Context, error) {
return ctx, nil
}
func (store *CassandraStore) CommitTransaction(ctx context.Context) error {
return nil
}
func (store *CassandraStore) RollbackTransaction(ctx context.Context) error {
return nil
}
func (store *CassandraStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
dir, name := entry.FullPath.DirAndName()
meta, err := entry.EncodeAttributesAndChunks()
@ -56,12 +67,12 @@ func (store *CassandraStore) InsertEntry(entry *filer2.Entry) (err error) {
return nil
}
func (store *CassandraStore) UpdateEntry(entry *filer2.Entry) (err error) {
func (store *CassandraStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
return store.InsertEntry(entry)
return store.InsertEntry(ctx, entry)
}
func (store *CassandraStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
func (store *CassandraStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
dir, name := fullpath.DirAndName()
var data []byte
@ -74,7 +85,7 @@ func (store *CassandraStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.
}
if len(data) == 0 {
return nil, fmt.Errorf("not found: %s", fullpath)
return nil, filer2.ErrNotFound
}
entry = &filer2.Entry{
@ -88,7 +99,7 @@ func (store *CassandraStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.
return entry, nil
}
func (store *CassandraStore) DeleteEntry(fullpath filer2.FullPath) error {
func (store *CassandraStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) error {
dir, name := fullpath.DirAndName()
@ -101,7 +112,7 @@ func (store *CassandraStore) DeleteEntry(fullpath filer2.FullPath) error {
return nil
}
func (store *CassandraStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool,
func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool,
limit int) (entries []*filer2.Entry, err error) {
cqlStr := "SELECT NAME, meta FROM filemeta WHERE directory=? AND name>? ORDER BY NAME ASC LIMIT ?"

13
weed/filer2/entry.go
View File

@ -52,9 +52,20 @@ func (entry *Entry) ToProtoEntry() *filer_pb.Entry {
return nil
}
return &filer_pb.Entry{
Name: string(entry.FullPath),
Name: entry.FullPath.Name(),
IsDirectory: entry.IsDirectory(),
Attributes: EntryAttributeToPb(entry),
Chunks: entry.Chunks,
}
}
func (entry *Entry) ToProtoFullEntry() *filer_pb.FullEntry {
if entry == nil {
return nil
}
dir, _ := entry.FullPath.DirAndName()
return &filer_pb.FullEntry{
Dir: dir,
Entry: entry.ToProtoEntry(),
}
}

2
weed/filer2/entry_codec.go
View File

@ -6,7 +6,7 @@ import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/gogo/protobuf/proto"
"github.com/golang/protobuf/proto"
)
func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {

16
weed/filer2/filechunks.go
View File

@ -40,7 +40,7 @@ func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*file
fileIds[interval.fileId] = true
}
for _, chunk := range chunks {
if found := fileIds[chunk.FileId]; found {
if _, found := fileIds[chunk.GetFileIdString()]; found {
compacted = append(compacted, chunk)
} else {
garbage = append(garbage, chunk)
@ -50,15 +50,15 @@ func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*file
return
}
func FindUnusedFileChunks(oldChunks, newChunks []*filer_pb.FileChunk) (unused []*filer_pb.FileChunk) {
func MinusChunks(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) {
fileIds := make(map[string]bool)
for _, interval := range newChunks {
fileIds[interval.FileId] = true
for _, interval := range bs {
fileIds[interval.GetFileIdString()] = true
}
for _, chunk := range oldChunks {
if found := fileIds[chunk.FileId]; !found {
unused = append(unused, chunk)
for _, chunk := range as {
if _, found := fileIds[chunk.GetFileIdString()]; !found {
delta = append(delta, chunk)
}
}
@ -123,7 +123,7 @@ func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.
newV := newVisibleInterval(
chunk.Offset,
chunk.Offset+int64(chunk.Size),
chunk.FileId,
chunk.GetFileIdString(),
chunk.Mtime,
true,
)

91
weed/filer2/filer.go
View File

@ -3,6 +3,7 @@ package filer2
import (
"context"
"fmt"
"google.golang.org/grpc"
"math"
"os"
"path/filepath"
@ -20,17 +21,19 @@ var (
)
type Filer struct {
store FilerStore
store *FilerStoreWrapper
directoryCache *ccache.Cache
MasterClient *wdclient.MasterClient
fileIdDeletionChan chan string
GrpcDialOption grpc.DialOption
}
func NewFiler(masters []string) *Filer {
func NewFiler(masters []string, grpcDialOption grpc.DialOption) *Filer {
f := &Filer{
directoryCache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
MasterClient: wdclient.NewMasterClient(context.Background(), "filer", masters),
MasterClient: wdclient.NewMasterClient(context.Background(), grpcDialOption, "filer", masters),
fileIdDeletionChan: make(chan string, 4096),
GrpcDialOption: grpcDialOption,
}
go f.loopProcessingDeletion()
@ -39,7 +42,7 @@ func NewFiler(masters []string) *Filer {
}
func (f *Filer) SetStore(store FilerStore) {
f.store = store
f.store = NewFilerStoreWrapper(store)
}
func (f *Filer) DisableDirectoryCache() {
@ -54,7 +57,19 @@ func (fs *Filer) KeepConnectedToMaster() {
fs.MasterClient.KeepConnectedToMaster()
}
func (f *Filer) CreateEntry(entry *Entry) error {
func (f *Filer) BeginTransaction(ctx context.Context) (context.Context, error) {
return f.store.BeginTransaction(ctx)
}
func (f *Filer) CommitTransaction(ctx context.Context) error {
return f.store.CommitTransaction(ctx)
}
func (f *Filer) RollbackTransaction(ctx context.Context) error {
return f.store.RollbackTransaction(ctx)
}
func (f *Filer) CreateEntry(ctx context.Context, entry *Entry) error {
if string(entry.FullPath) == "/" {
return nil
@ -67,7 +82,7 @@ func (f *Filer) CreateEntry(entry *Entry) error {
var lastDirectoryEntry *Entry
for i := 1; i < len(dirParts); i++ {
dirPath := "/" + filepath.Join(dirParts[:i]...)
dirPath := "/" + filepath.ToSlash(filepath.Join(dirParts[:i]...))
// fmt.Printf("%d directory: %+v\n", i, dirPath)
// first check local cache
@ -76,7 +91,7 @@ func (f *Filer) CreateEntry(entry *Entry) error {
// not found, check the store directly
if dirEntry == nil {
glog.V(4).Infof("find uncached directory: %s", dirPath)
dirEntry, _ = f.FindEntry(FullPath(dirPath))
dirEntry, _ = f.FindEntry(ctx, FullPath(dirPath))
} else {
glog.V(4).Infof("found cached directory: %s", dirPath)
}
@ -99,9 +114,9 @@ func (f *Filer) CreateEntry(entry *Entry) error {
}
glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode)
mkdirErr := f.store.InsertEntry(dirEntry)
mkdirErr := f.store.InsertEntry(ctx, dirEntry)
if mkdirErr != nil {
if _, err := f.FindEntry(FullPath(dirPath)); err == ErrNotFound {
if _, err := f.FindEntry(ctx, FullPath(dirPath)); err == ErrNotFound {
return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
}
} else {
@ -134,14 +149,16 @@ func (f *Filer) CreateEntry(entry *Entry) error {
}
*/
oldEntry, _ := f.FindEntry(entry.FullPath)
oldEntry, _ := f.FindEntry(ctx, entry.FullPath)
if oldEntry == nil {
if err := f.store.InsertEntry(entry); err != nil {
if err := f.store.InsertEntry(ctx, entry); err != nil {
glog.Errorf("insert entry %s: %v", entry.FullPath, err)
return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
}
} else {
if err := f.UpdateEntry(oldEntry, entry); err != nil {
if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
glog.Errorf("update entry %s: %v", entry.FullPath, err)
return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
}
}
@ -153,19 +170,21 @@ func (f *Filer) CreateEntry(entry *Entry) error {
return nil
}
func (f *Filer) UpdateEntry(oldEntry, entry *Entry) (err error) {
func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err error) {
if oldEntry != nil {
if oldEntry.IsDirectory() && !entry.IsDirectory() {
glog.Errorf("existing %s is a directory", entry.FullPath)
return fmt.Errorf("existing %s is a directory", entry.FullPath)
}
if !oldEntry.IsDirectory() && entry.IsDirectory() {
glog.Errorf("existing %s is a file", entry.FullPath)
return fmt.Errorf("existing %s is a file", entry.FullPath)
}
}
return f.store.UpdateEntry(entry)
return f.store.UpdateEntry(ctx, entry)
}
func (f *Filer) FindEntry(p FullPath) (entry *Entry, err error) {
func (f *Filer) FindEntry(ctx context.Context, p FullPath) (entry *Entry, err error) {
now := time.Now()
@ -181,11 +200,11 @@ func (f *Filer) FindEntry(p FullPath) (entry *Entry, err error) {
},
}, nil
}
return f.store.FindEntry(p)
return f.store.FindEntry(ctx, p)
}
func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDeleteChunks bool) (err error) {
entry, err := f.FindEntry(p)
func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p FullPath, isRecursive bool, shouldDeleteChunks bool) (err error) {
entry, err := f.FindEntry(ctx, p)
if err != nil {
return err
}
@ -198,37 +217,41 @@ func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDelet
lastFileName := ""
includeLastFile := false
for limit > 0 {
entries, err := f.ListDirectoryEntries(p, lastFileName, includeLastFile, 1024)
entries, err := f.ListDirectoryEntries(ctx, p, lastFileName, includeLastFile, 1024)
if err != nil {
glog.Errorf("list folder %s: %v", p, err)
return fmt.Errorf("list folder %s: %v", p, err)
}
if len(entries) == 0 {
break
} else {
}
if isRecursive {
for _, sub := range entries {
lastFileName = sub.Name()
f.DeleteEntryMetaAndData(sub.FullPath, isRecursive, shouldDeleteChunks)
err = f.DeleteEntryMetaAndData(ctx, sub.FullPath, isRecursive, shouldDeleteChunks)
if err != nil {
return err
}
limit--
if limit <= 0 {
break
}
}
} else {
if len(entries) > 0 {
return fmt.Errorf("folder %s is not empty", p)
}
}
f.cacheDelDirectory(string(p))
if len(entries) < 1024 {
break
}
}
}
f.cacheDelDirectory(string(p))
}
if shouldDeleteChunks {
f.DeleteChunks(entry.Chunks)
f.DeleteChunks(p, entry.Chunks)
}
if p == "/" {
@ -238,17 +261,22 @@ func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDelet
f.NotifyUpdateEvent(entry, nil, shouldDeleteChunks)
return f.store.DeleteEntry(p)
return f.store.DeleteEntry(ctx, p)
}
func (f *Filer) ListDirectoryEntries(p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) {
func (f *Filer) ListDirectoryEntries(ctx context.Context, p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) {
if strings.HasSuffix(string(p), "/") && len(p) > 1 {
p = p[0 : len(p)-1]
}
return f.store.ListDirectoryEntries(p, startFileName, inclusive, limit)
return f.store.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit)
}
func (f *Filer) cacheDelDirectory(dirpath string) {
if dirpath == "/" {
return
}
if f.directoryCache == nil {
return
}
@ -257,6 +285,7 @@ func (f *Filer) cacheDelDirectory(dirpath string) {
}
func (f *Filer) cacheGetDirectory(dirpath string) *Entry {
if f.directoryCache == nil {
return nil
}

163
weed/filer2/filer_client_util.go Normal file
View File

@ -0,0 +1,163 @@
package filer2
import (
"context"
"fmt"
"strings"
"sync"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
)
func VolumeId(fileId string) string {
lastCommaIndex := strings.LastIndex(fileId, ",")
if lastCommaIndex > 0 {
return fileId[:lastCommaIndex]
}
return fileId
}
type FilerClient interface {
WithFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error
}
func ReadIntoBuffer(ctx context.Context, filerClient FilerClient, fullFilePath string, buff []byte, chunkViews []*ChunkView, baseOffset int64) (totalRead int64, err error) {
var vids []string
for _, chunkView := range chunkViews {
vids = append(vids, VolumeId(chunkView.FileId))
}
vid2Locations := make(map[string]*filer_pb.Locations)
err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
glog.V(4).Infof("read fh lookup volume id locations: %v", vids)
resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
VolumeIds: vids,
})
if err != nil {
return err
}
vid2Locations = resp.LocationsMap
return nil
})
if err != nil {
return 0, fmt.Errorf("failed to lookup volume ids %v: %v", vids, err)
}
var wg sync.WaitGroup
for _, chunkView := range chunkViews {
wg.Add(1)
go func(chunkView *ChunkView) {
defer wg.Done()
glog.V(4).Infof("read fh reading chunk: %+v", chunkView)
locations := vid2Locations[VolumeId(chunkView.FileId)]
if locations == nil || len(locations.Locations) == 0 {
glog.V(0).Infof("failed to locate %s", chunkView.FileId)
err = fmt.Errorf("failed to locate %s", chunkView.FileId)
return
}
var n int64
n, err = util.ReadUrl(
fmt.Sprintf("http://%s/%s", locations.Locations[0].Url, chunkView.FileId),
chunkView.Offset,
int(chunkView.Size),
buff[chunkView.LogicOffset-baseOffset:chunkView.LogicOffset-baseOffset+int64(chunkView.Size)],
!chunkView.IsFullChunk)
if err != nil {
glog.V(0).Infof("%v read http://%s/%v %v bytes: %v", fullFilePath, locations.Locations[0].Url, chunkView.FileId, n, err)
err = fmt.Errorf("failed to read http://%s/%s: %v",
locations.Locations[0].Url, chunkView.FileId, err)
return
}
glog.V(4).Infof("read fh read %d bytes: %+v", n, chunkView)
totalRead += n
}(chunkView)
}
wg.Wait()
return
}
func GetEntry(ctx context.Context, filerClient FilerClient, fullFilePath string) (entry *filer_pb.Entry, err error) {
dir, name := FullPath(fullFilePath).DirAndName()
err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
}
glog.V(3).Infof("read %s request: %v", fullFilePath, request)
resp, err := client.LookupDirectoryEntry(ctx, request)
if err != nil {
if err == ErrNotFound || strings.Contains(err.Error(), ErrNotFound.Error()) {
return nil
}
glog.V(3).Infof("read %s attr %v: %v", fullFilePath, request, err)
return err
}
if resp.Entry != nil {
entry = resp.Entry
}
return nil
})
return
}
func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath string, fn func(entry *filer_pb.Entry)) (err error) {
err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
paginationLimit := 1024
lastEntryName := ""
for {
request := &filer_pb.ListEntriesRequest{
Directory: fullDirPath,
StartFromFileName: lastEntryName,
Limit: uint32(paginationLimit),
}
glog.V(3).Infof("read directory: %v", request)
resp, err := client.ListEntries(ctx, request)
if err != nil {
return fmt.Errorf("list %s: %v", fullDirPath, err)
}
for _, entry := range resp.Entries {
fn(entry)
lastEntryName = entry.Name
}
if len(resp.Entries) < paginationLimit {
break
}
}
return nil
})
return
}

28
weed/filer2/filer_deletion.go
View File

@ -38,25 +38,28 @@ func (f *Filer) loopProcessingDeletion() {
fileIds = append(fileIds, fid)
if len(fileIds) >= 4096 {
glog.V(1).Infof("deleting fileIds len=%d", len(fileIds))
operation.DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, fileIds, lookupFunc)
fileIds = fileIds[:0]
}
case <-ticker.C:
if len(fileIds) > 0 {
glog.V(1).Infof("timed deletion fileIds len=%d", len(fileIds))
operation.DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, fileIds, lookupFunc)
fileIds = fileIds[:0]
}
}
}
}
func (f *Filer) DeleteChunks(chunks []*filer_pb.FileChunk) {
func (f *Filer) DeleteChunks(fullpath FullPath, chunks []*filer_pb.FileChunk) {
for _, chunk := range chunks {
f.fileIdDeletionChan <- chunk.FileId
glog.V(3).Infof("deleting %s chunk %s", fullpath, chunk.String())
f.fileIdDeletionChan <- chunk.GetFileIdString()
}
}
// DeleteFileByFileId direct delete by file id.
// Only used when the fileId is not being managed by snapshots.
func (f *Filer) DeleteFileByFileId(fileId string) {
f.fileIdDeletionChan <- fileId
}
@ -67,22 +70,19 @@ func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {
return
}
if newEntry == nil {
f.DeleteChunks(oldEntry.Chunks)
f.DeleteChunks(oldEntry.FullPath, oldEntry.Chunks)
}
var toDelete []*filer_pb.FileChunk
newChunkIds := make(map[string]bool)
for _, newChunk := range newEntry.Chunks {
newChunkIds[newChunk.GetFileIdString()] = true
}
for _, oldChunk := range oldEntry.Chunks {
found := false
for _, newChunk := range newEntry.Chunks {
if oldChunk.FileId == newChunk.FileId {
found = true
break
}
}
if !found {
if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found {
toDelete = append(toDelete, oldChunk)
}
}
f.DeleteChunks(toDelete)
f.DeleteChunks(oldEntry.FullPath, toDelete)
}

6
weed/filer2/filer_notify.go
View File

@ -20,12 +20,18 @@ func (f *Filer) NotifyUpdateEvent(oldEntry, newEntry *Entry, deleteChunks bool)
glog.V(3).Infof("notifying entry update %v", key)
newParentPath := ""
if newEntry != nil {
newParentPath, _ = newEntry.FullPath.DirAndName()
}
notification.Queue.SendMessage(
key,
&filer_pb.EventNotification{
OldEntry: oldEntry.ToProtoEntry(),
NewEntry: newEntry.ToProtoEntry(),
DeleteChunks: deleteChunks,
NewParentPath: newParentPath,
},
)

113
weed/filer2/filerstore.go
View File

@ -1,7 +1,12 @@
package filer2
import (
"context"
"errors"
"time"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util"
)
@ -10,12 +15,110 @@ type FilerStore interface {
GetName() string
// Initialize initializes the file store
Initialize(configuration util.Configuration) error
InsertEntry(*Entry) error
UpdateEntry(*Entry) (err error)
InsertEntry(context.Context, *Entry) error
UpdateEntry(context.Context, *Entry) (err error)
// err == filer2.ErrNotFound if not found
FindEntry(FullPath) (entry *Entry, err error)
DeleteEntry(FullPath) (err error)
ListDirectoryEntries(dirPath FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error)
FindEntry(context.Context, FullPath) (entry *Entry, err error)
DeleteEntry(context.Context, FullPath) (err error)
ListDirectoryEntries(ctx context.Context, dirPath FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error)
BeginTransaction(ctx context.Context) (context.Context, error)
CommitTransaction(ctx context.Context) error
RollbackTransaction(ctx context.Context) error
}
var ErrNotFound = errors.New("filer: no entry is found in filer store")
type FilerStoreWrapper struct {
actualStore FilerStore
}
func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper {
return &FilerStoreWrapper{
actualStore: store,
}
}
func (fsw *FilerStoreWrapper) GetName() string {
return fsw.actualStore.GetName()
}
func (fsw *FilerStoreWrapper) Initialize(configuration util.Configuration) error {
return fsw.actualStore.Initialize(configuration)
}
func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) error {
stats.FilerStoreCounter.WithLabelValues(fsw.actualStore.GetName(), "insert").Inc()
start := time.Now()
defer func() {
stats.FilerStoreHistogram.WithLabelValues(fsw.actualStore.GetName(), "insert").Observe(time.Since(start).Seconds())
}()
filer_pb.BeforeEntrySerialization(entry.Chunks)
return fsw.actualStore.InsertEntry(ctx, entry)
}
func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) error {
stats.FilerStoreCounter.WithLabelValues(fsw.actualStore.GetName(), "update").Inc()
start := time.Now()
defer func() {
stats.FilerStoreHistogram.WithLabelValues(fsw.actualStore.GetName(), "update").Observe(time.Since(start).Seconds())
}()
filer_pb.BeforeEntrySerialization(entry.Chunks)
return fsw.actualStore.UpdateEntry(ctx, entry)
}
func (fsw *FilerStoreWrapper) FindEntry(ctx context.Context, fp FullPath) (entry *Entry, err error) {
stats.FilerStoreCounter.WithLabelValues(fsw.actualStore.GetName(), "find").Inc()
start := time.Now()
defer func() {
stats.FilerStoreHistogram.WithLabelValues(fsw.actualStore.GetName(), "find").Observe(time.Since(start).Seconds())
}()
entry, err = fsw.actualStore.FindEntry(ctx, fp)
if err != nil {
return nil, err
}
filer_pb.AfterEntryDeserialization(entry.Chunks)
return
}
func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp FullPath) (err error) {
stats.FilerStoreCounter.WithLabelValues(fsw.actualStore.GetName(), "delete").Inc()
start := time.Now()
defer func() {
stats.FilerStoreHistogram.WithLabelValues(fsw.actualStore.GetName(), "delete").Observe(time.Since(start).Seconds())
}()
return fsw.actualStore.DeleteEntry(ctx, fp)
}
func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) {
stats.FilerStoreCounter.WithLabelValues(fsw.actualStore.GetName(), "list").Inc()
start := time.Now()
defer func() {
stats.FilerStoreHistogram.WithLabelValues(fsw.actualStore.GetName(), "list").Observe(time.Since(start).Seconds())
}()
entries, err := fsw.actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit)
if err != nil {
return nil, err
}
for _, entry := range entries {
filer_pb.AfterEntryDeserialization(entry.Chunks)
}
return entries, err
}
func (fsw *FilerStoreWrapper) BeginTransaction(ctx context.Context) (context.Context, error) {
return fsw.actualStore.BeginTransaction(ctx)
}
func (fsw *FilerStoreWrapper) CommitTransaction(ctx context.Context) error {
return fsw.actualStore.CommitTransaction(ctx)
}
func (fsw *FilerStoreWrapper) RollbackTransaction(ctx context.Context) error {
return fsw.actualStore.RollbackTransaction(ctx)
}

13
weed/filer2/fullpath.go
View File

@ -8,10 +8,7 @@ import (
type FullPath string
func NewFullPath(dir, name string) FullPath {
if strings.HasSuffix(dir, "/") {
return FullPath(dir + name)
}
return FullPath(dir + "/" + name)
return FullPath(dir).Child(name)
}
func (fp FullPath) DirAndName() (string, string) {
@ -29,3 +26,11 @@ func (fp FullPath) Name() string {
_, name := filepath.Split(string(fp))
return name
}
func (fp FullPath) Child(name string) FullPath {
dir := string(fp)
if strings.HasSuffix(dir, "/") {
return FullPath(dir + name)
}
return FullPath(dir + "/" + name)
}

32
weed/filer2/leveldb/leveldb_store.go
View File

@ -2,12 +2,14 @@ package leveldb
import (
"bytes"
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
weed_util "github.com/chrislusf/seaweedfs/weed/util"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/opt"
leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
)
@ -38,14 +40,30 @@ func (store *LevelDBStore) initialize(dir string) (err error) {
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
}
if store.db, err = leveldb.OpenFile(dir, nil); err != nil {
opts := &opt.Options{
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
CompactionTableSizeMultiplier: 10,
}
if store.db, err = leveldb.OpenFile(dir, opts); err != nil {
glog.Infof("filer store open dir %s: %v", dir, err)
return
}
return
}
func (store *LevelDBStore) InsertEntry(entry *filer2.Entry) (err error) {
func (store *LevelDBStore) BeginTransaction(ctx context.Context) (context.Context, error) {
return ctx, nil
}
func (store *LevelDBStore) CommitTransaction(ctx context.Context) error {
return nil
}
func (store *LevelDBStore) RollbackTransaction(ctx context.Context) error {
return nil
}
func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
key := genKey(entry.DirAndName())
value, err := entry.EncodeAttributesAndChunks()
@ -64,12 +82,12 @@ func (store *LevelDBStore) InsertEntry(entry *filer2.Entry) (err error) {
return nil
}
func (store *LevelDBStore) UpdateEntry(entry *filer2.Entry) (err error) {
func (store *LevelDBStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
return store.InsertEntry(entry)
return store.InsertEntry(ctx, entry)
}
func (store *LevelDBStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
func (store *LevelDBStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
key := genKey(fullpath.DirAndName())
data, err := store.db.Get(key, nil)
@ -94,7 +112,7 @@ func (store *LevelDBStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.En
return entry, nil
}
func (store *LevelDBStore) DeleteEntry(fullpath filer2.FullPath) (err error) {
func (store *LevelDBStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) {
key := genKey(fullpath.DirAndName())
err = store.db.Delete(key, nil)
@ -105,7 +123,7 @@ func (store *LevelDBStore) DeleteEntry(fullpath filer2.FullPath) (err error) {
return nil
}
func (store *LevelDBStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool,
func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool,
limit int) (entries []*filer2.Entry, err error) {
directoryPrefix := genDirectoryKeyPrefix(fullpath, "")

19
weed/filer2/leveldb/leveldb_store_test.go
View File

@ -1,6 +1,7 @@
package leveldb
import (
"context"
"github.com/chrislusf/seaweedfs/weed/filer2"
"io/ioutil"
"os"
@ -8,7 +9,7 @@ import (
)
func TestCreateAndFind(t *testing.T) {
filer := filer2.NewFiler(nil)
filer := filer2.NewFiler(nil, nil)
dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
defer os.RemoveAll(dir)
store := &LevelDBStore{}
@ -18,6 +19,8 @@ func TestCreateAndFind(t *testing.T) {
fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")
ctx := context.Background()
entry1 := &filer2.Entry{
FullPath: fullpath,
Attr: filer2.Attr{
@ -27,12 +30,12 @@ func TestCreateAndFind(t *testing.T) {
},
}
if err := filer.CreateEntry(entry1); err != nil {
if err := filer.CreateEntry(ctx, entry1); err != nil {
t.Errorf("create entry %v: %v", entry1.FullPath, err)
return
}
entry, err := filer.FindEntry(fullpath)
entry, err := filer.FindEntry(ctx, fullpath)
if err != nil {
t.Errorf("find entry: %v", err)
@ -45,14 +48,14 @@ func TestCreateAndFind(t *testing.T) {
}
// checking one upper directory
entries, _ := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one"), "", false, 100)
entries, _ := filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one"), "", false, 100)
if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries))
return
}
// checking one upper directory
entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100)
entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries))
return
@ -61,7 +64,7 @@ func TestCreateAndFind(t *testing.T) {
}
func TestEmptyRoot(t *testing.T) {
filer := filer2.NewFiler(nil)
filer := filer2.NewFiler(nil, nil)
dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
defer os.RemoveAll(dir)
store := &LevelDBStore{}
@ -69,8 +72,10 @@ func TestEmptyRoot(t *testing.T) {
filer.SetStore(store)
filer.DisableDirectoryCache()
ctx := context.Background()
// checking one upper directory
entries, err := filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100)
entries, err := filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
if err != nil {
t.Errorf("list entries: %v", err)
return

208
weed/filer2/leveldb2/leveldb2_store.go Normal file
View File

@ -0,0 +1,208 @@
package leveldb
import (
"bytes"
"context"
"crypto/md5"
"fmt"
"io"
"os"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
weed_util "github.com/chrislusf/seaweedfs/weed/util"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/opt"
leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
)
func init() {
filer2.Stores = append(filer2.Stores, &LevelDB2Store{})
}
type LevelDB2Store struct {
dbs []*leveldb.DB
dbCount int
}
func (store *LevelDB2Store) GetName() string {
return "leveldb2"
}
func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration) (err error) {
dir := configuration.GetString("dir")
return store.initialize(dir, 8)
}
func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
glog.Infof("filer store leveldb2 dir: %s", dir)
if err := weed_util.TestFolderWritable(dir); err != nil {
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
}
opts := &opt.Options{
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
CompactionTableSizeMultiplier: 4,
}
for d := 0 ; d < dbCount; d++ {
dbFolder := fmt.Sprintf("%s/%02d", dir, d)
os.MkdirAll(dbFolder, 0755)
db, dbErr := leveldb.OpenFile(dbFolder, opts)
if dbErr != nil {
glog.Errorf("filer store open dir %s: %v", dbFolder, dbErr)
return
}
store.dbs = append(store.dbs, db)
}
store.dbCount = dbCount
return
}
func (store *LevelDB2Store) BeginTransaction(ctx context.Context) (context.Context, error) {
return ctx, nil
}
func (store *LevelDB2Store) CommitTransaction(ctx context.Context) error {
return nil
}
func (store *LevelDB2Store) RollbackTransaction(ctx context.Context) error {
return nil
}
func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
dir, name := entry.DirAndName()
key, partitionId := genKey(dir, name, store.dbCount)
value, err := entry.EncodeAttributesAndChunks()
if err != nil {
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
}
err = store.dbs[partitionId].Put(key, value, nil)
if err != nil {
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
}
// println("saved", entry.FullPath, "chunks", len(entry.Chunks))
return nil
}
func (store *LevelDB2Store) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
return store.InsertEntry(ctx, entry)
}
func (store *LevelDB2Store) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
dir, name := fullpath.DirAndName()
key, partitionId := genKey(dir, name, store.dbCount)
data, err := store.dbs[partitionId].Get(key, nil)
if err == leveldb.ErrNotFound {
return nil, filer2.ErrNotFound
}
if err != nil {
return nil, fmt.Errorf("get %s : %v", entry.FullPath, err)
}
entry = &filer2.Entry{
FullPath: fullpath,
}
err = entry.DecodeAttributesAndChunks(data)
if err != nil {
return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
}
// println("read", entry.FullPath, "chunks", len(entry.Chunks), "data", len(data), string(data))
return entry, nil
}
func (store *LevelDB2Store) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) {
dir, name := fullpath.DirAndName()
key, partitionId := genKey(dir, name, store.dbCount)
err = store.dbs[partitionId].Delete(key, nil)
if err != nil {
return fmt.Errorf("delete %s : %v", fullpath, err)
}
return nil
}
func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool,
limit int) (entries []*filer2.Entry, err error) {
directoryPrefix, partitionId := genDirectoryKeyPrefix(fullpath, "", store.dbCount)
lastFileStart, _ := genDirectoryKeyPrefix(fullpath, startFileName, store.dbCount)
iter := store.dbs[partitionId].NewIterator(&leveldb_util.Range{Start: lastFileStart}, nil)
for iter.Next() {
key := iter.Key()
if !bytes.HasPrefix(key, directoryPrefix) {
break
}
fileName := getNameFromKey(key)
if fileName == "" {
continue
}
if fileName == startFileName && !inclusive {
continue
}
limit--
if limit < 0 {
break
}
entry := &filer2.Entry{
FullPath: filer2.NewFullPath(string(fullpath), fileName),
}
// println("list", entry.FullPath, "chunks", len(entry.Chunks))
if decodeErr := entry.DecodeAttributesAndChunks(iter.Value()); decodeErr != nil {
err = decodeErr
glog.V(0).Infof("list %s : %v", entry.FullPath, err)
break
}
entries = append(entries, entry)
}
iter.Release()
return entries, err
}
func genKey(dirPath, fileName string, dbCount int) (key []byte, partitionId int) {
key, partitionId = hashToBytes(dirPath, dbCount)
key = append(key, []byte(fileName)...)
return key, partitionId
}
func genDirectoryKeyPrefix(fullpath filer2.FullPath, startFileName string, dbCount int) (keyPrefix []byte, partitionId int) {
keyPrefix, partitionId = hashToBytes(string(fullpath), dbCount)
if len(startFileName) > 0 {
keyPrefix = append(keyPrefix, []byte(startFileName)...)
}
return keyPrefix, partitionId
}
func getNameFromKey(key []byte) string {
return string(key[md5.Size:])
}
// hash directory, and use last byte for partitioning
func hashToBytes(dir string, dbCount int) ([]byte, int) {
h := md5.New()
io.WriteString(h, dir)
b := h.Sum(nil)
x := b[len(b)-1]
return b, int(x)%dbCount
}

88
weed/filer2/leveldb2/leveldb2_store_test.go Normal file
View File

@ -0,0 +1,88 @@
package leveldb
import (
"context"
"github.com/chrislusf/seaweedfs/weed/filer2"
"io/ioutil"
"os"
"testing"
)
func TestCreateAndFind(t *testing.T) {
filer := filer2.NewFiler(nil, nil)
dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
defer os.RemoveAll(dir)
store := &LevelDB2Store{}
store.initialize(dir,2)
filer.SetStore(store)
filer.DisableDirectoryCache()
fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")
ctx := context.Background()
entry1 := &filer2.Entry{
FullPath: fullpath,
Attr: filer2.Attr{
Mode: 0440,
Uid: 1234,
Gid: 5678,
},
}
if err := filer.CreateEntry(ctx, entry1); err != nil {
t.Errorf("create entry %v: %v", entry1.FullPath, err)
return
}
entry, err := filer.FindEntry(ctx, fullpath)
if err != nil {
t.Errorf("find entry: %v", err)
return
}
if entry.FullPath != entry1.FullPath {
t.Errorf("find wrong entry: %v", entry.FullPath)
return
}
// checking one upper directory
entries, _ := filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one"), "", false, 100)
if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries))
return
}
// checking one upper directory
entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries))
return
}
}
func TestEmptyRoot(t *testing.T) {
filer := filer2.NewFiler(nil, nil)
dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
defer os.RemoveAll(dir)
store := &LevelDB2Store{}
store.initialize(dir,2)
filer.SetStore(store)
filer.DisableDirectoryCache()
ctx := context.Background()
// checking one upper directory
entries, err := filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
if err != nil {
t.Errorf("list entries: %v", err)
return
}
if len(entries) != 0 {
t.Errorf("list entries count: %v", len(entries))
return
}
}

35
weed/filer2/memdb/memdb_store.go
View File

@ -1,11 +1,13 @@
package memdb
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/google/btree"
"strings"
"sync"
)
func init() {
@ -14,6 +16,7 @@ func init() {
type MemDbStore struct {
tree *btree.BTree
treeLock sync.Mutex
}
type entryItem struct {
@ -33,21 +36,35 @@ func (store *MemDbStore) Initialize(configuration util.Configuration) (err error
return nil
}
func (store *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) {
// println("inserting", entry.FullPath)
store.tree.ReplaceOrInsert(entryItem{entry})
func (store *MemDbStore) BeginTransaction(ctx context.Context) (context.Context, error) {
return ctx, nil
}
func (store *MemDbStore) CommitTransaction(ctx context.Context) error {
return nil
}
func (store *MemDbStore) RollbackTransaction(ctx context.Context) error {
return nil
}
func (store *MemDbStore) UpdateEntry(entry *filer2.Entry) (err error) {
if _, err = store.FindEntry(entry.FullPath); err != nil {
func (store *MemDbStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
// println("inserting", entry.FullPath)
store.treeLock.Lock()
store.tree.ReplaceOrInsert(entryItem{entry})
store.treeLock.Unlock()
return nil
}
func (store *MemDbStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
if _, err = store.FindEntry(ctx, entry.FullPath); err != nil {
return fmt.Errorf("no such file %s : %v", entry.FullPath, err)
}
store.treeLock.Lock()
store.tree.ReplaceOrInsert(entryItem{entry})
store.treeLock.Unlock()
return nil
}
func (store *MemDbStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
func (store *MemDbStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
item := store.tree.Get(entryItem{&filer2.Entry{FullPath: fullpath}})
if item == nil {
return nil, filer2.ErrNotFound
@ -56,12 +73,14 @@ func (store *MemDbStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entr
return entry, nil
}
func (store *MemDbStore) DeleteEntry(fullpath filer2.FullPath) (err error) {
func (store *MemDbStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) {
store.treeLock.Lock()
store.tree.Delete(entryItem{&filer2.Entry{FullPath: fullpath}})
store.treeLock.Unlock()
return nil
}
func (store *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
func (store *MemDbStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
startFrom := string(fullpath)
if startFileName != "" {

33
weed/filer2/memdb/memdb_store_test.go
View File

@ -1,17 +1,20 @@
package memdb
import (
"context"
"github.com/chrislusf/seaweedfs/weed/filer2"
"testing"
)
func TestCreateAndFind(t *testing.T) {
filer := filer2.NewFiler(nil)
filer := filer2.NewFiler(nil, nil)
store := &MemDbStore{}
store.Initialize(nil)
filer.SetStore(store)
filer.DisableDirectoryCache()
ctx := context.Background()
fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")
entry1 := &filer2.Entry{
@ -23,12 +26,12 @@ func TestCreateAndFind(t *testing.T) {
},
}
if err := filer.CreateEntry(entry1); err != nil {
if err := filer.CreateEntry(ctx, entry1); err != nil {
t.Errorf("create entry %v: %v", entry1.FullPath, err)
return
}
entry, err := filer.FindEntry(fullpath)
entry, err := filer.FindEntry(ctx, fullpath)
if err != nil {
t.Errorf("find entry: %v", err)
@ -43,12 +46,14 @@ func TestCreateAndFind(t *testing.T) {
}
func TestCreateFileAndList(t *testing.T) {
filer := filer2.NewFiler(nil)
filer := filer2.NewFiler(nil, nil)
store := &MemDbStore{}
store.Initialize(nil)
filer.SetStore(store)
filer.DisableDirectoryCache()
ctx := context.Background()
entry1 := &filer2.Entry{
FullPath: filer2.FullPath("/home/chris/this/is/one/file1.jpg"),
Attr: filer2.Attr{
@ -67,11 +72,11 @@ func TestCreateFileAndList(t *testing.T) {
},
}
filer.CreateEntry(entry1)
filer.CreateEntry(entry2)
filer.CreateEntry(ctx, entry1)
filer.CreateEntry(ctx, entry2)
// checking the 2 files
entries, err := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/"), "", false, 100)
entries, err := filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one/"), "", false, 100)
if err != nil {
t.Errorf("list entries: %v", err)
@ -94,21 +99,21 @@ func TestCreateFileAndList(t *testing.T) {
}
// checking the offset
entries, err = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/"), "file1.jpg", false, 100)
entries, err = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one/"), "file1.jpg", false, 100)
if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries))
return
}
// checking one upper directory
entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100)
entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is"), "", false, 100)
if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries))
return
}
// checking root directory
entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100)
entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries))
return
@ -124,18 +129,18 @@ func TestCreateFileAndList(t *testing.T) {
Gid: 5678,
},
}
filer.CreateEntry(entry3)
filer.CreateEntry(ctx, entry3)
// checking one upper directory
entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100)
entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is"), "", false, 100)
if len(entries) != 2 {
t.Errorf("list entries count: %v", len(entries))
return
}
// delete file and count
filer.DeleteEntryMetaAndData(file3Path, false, false)
entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100)
filer.DeleteEntryMetaAndData(ctx, file3Path, false, false)
entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is"), "", false, 100)
if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries))
return

4
weed/filer2/postgres/README.txt
View File

@ -9,8 +9,8 @@ $PGHOME/bin/psql --username=postgres --password seaweedfs
CREATE TABLE IF NOT EXISTS filemeta (
dirhash BIGINT,
name VARCHAR(1000),
directory VARCHAR(4096),
name VARCHAR(65535),
directory VARCHAR(65535),
meta bytea,
PRIMARY KEY (dirhash, name)
);

4
weed/filer2/redis/redis_cluster_store.go
View File

@ -21,12 +21,14 @@ func (store *RedisClusterStore) GetName() string {
func (store *RedisClusterStore) Initialize(configuration util.Configuration) (err error) {
return store.initialize(
configuration.GetStringSlice("addresses"),
configuration.GetString("password"),
)
}
func (store *RedisClusterStore) initialize(addresses []string) (err error) {
func (store *RedisClusterStore) initialize(addresses []string, password string) (err error) {
store.Client = redis.NewClusterClient(&redis.ClusterOptions{
Addrs: addresses,
Password: password,
})
return
}

25
weed/filer2/redis/universal_redis_store.go
View File

@ -1,6 +1,7 @@
package redis
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
@ -18,7 +19,17 @@ type UniversalRedisStore struct {
Client redis.UniversalClient
}
func (store *UniversalRedisStore) InsertEntry(entry *filer2.Entry) (err error) {
func (store *UniversalRedisStore) BeginTransaction(ctx context.Context) (context.Context, error) {
return ctx, nil
}
func (store *UniversalRedisStore) CommitTransaction(ctx context.Context) error {
return nil
}
func (store *UniversalRedisStore) RollbackTransaction(ctx context.Context) error {
return nil
}
func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
value, err := entry.EncodeAttributesAndChunks()
if err != nil {
@ -42,12 +53,12 @@ func (store *UniversalRedisStore) InsertEntry(entry *filer2.Entry) (err error) {
return nil
}
func (store *UniversalRedisStore) UpdateEntry(entry *filer2.Entry) (err error) {
func (store *UniversalRedisStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
return store.InsertEntry(entry)
return store.InsertEntry(ctx, entry)
}
func (store *UniversalRedisStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
data, err := store.Client.Get(string(fullpath)).Result()
if err == redis.Nil {
@ -69,7 +80,7 @@ func (store *UniversalRedisStore) FindEntry(fullpath filer2.FullPath) (entry *fi
return entry, nil
}
func (store *UniversalRedisStore) DeleteEntry(fullpath filer2.FullPath) (err error) {
func (store *UniversalRedisStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) {
_, err = store.Client.Del(string(fullpath)).Result()
@ -88,7 +99,7 @@ func (store *UniversalRedisStore) DeleteEntry(fullpath filer2.FullPath) (err err
return nil
}
func (store *UniversalRedisStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool,
func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool,
limit int) (entries []*filer2.Entry, err error) {
members, err := store.Client.SMembers(genDirectoryListKey(string(fullpath))).Result()
@ -126,7 +137,7 @@ func (store *UniversalRedisStore) ListDirectoryEntries(fullpath filer2.FullPath,
// fetch entry meta
for _, fileName := range members {
path := filer2.NewFullPath(string(fullpath), fileName)
entry, err := store.FindEntry(path)
entry, err := store.FindEntry(ctx, path)
if err != nil {
glog.V(0).Infof("list %s : %v", path, err)
} else {

41
weed/filer2/stream.go Normal file
View File

@ -0,0 +1,41 @@
package filer2
import (
"io"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/wdclient"
)
func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int) error {
chunkViews := ViewFromChunks(chunks, offset, size)
fileId2Url := make(map[string]string)
for _, chunkView := range chunkViews {
urlString, err := masterClient.LookupFileId(chunkView.FileId)
if err != nil {
glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
return err
}
fileId2Url[chunkView.FileId] = urlString
}
for _, chunkView := range chunkViews {
urlString := fileId2Url[chunkView.FileId]
_, err := util.ReadUrlAsStream(urlString, chunkView.Offset, int(chunkView.Size), func(data []byte) {
w.Write(data)
})
if err != nil {
glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err)
return err
}
}
return nil
}

113
weed/filesys/dir.go
View File

@ -4,7 +4,6 @@ import (
"context"
"os"
"path"
"path/filepath"
"time"
"github.com/chrislusf/seaweedfs/weed/filer2"
@ -29,15 +28,13 @@ var _ = fs.NodeRemover(&Dir{})
var _ = fs.NodeRenamer(&Dir{})
var _ = fs.NodeSetattrer(&Dir{})
func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error {
// https://github.com/bazil/fuse/issues/196
attr.Valid = time.Second
if dir.Path == dir.wfs.option.FilerMountRootPath {
attr.Uid = dir.wfs.option.MountUid
attr.Gid = dir.wfs.option.MountGid
attr.Mode = dir.wfs.option.MountMode
dir.setRootDirAttributes(attr)
return nil
}
@ -54,40 +51,14 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
return nil
}
parent, name := filepath.Split(dir.Path)
err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.LookupDirectoryEntryRequest{
Directory: parent,
Name: name,
}
glog.V(1).Infof("read dir %s attr: %v", dir.Path, request)
resp, err := client.LookupDirectoryEntry(context, request)
entry, err := filer2.GetEntry(ctx, dir.wfs, dir.Path)
if err != nil {
if err == filer2.ErrNotFound {
return nil
}
glog.V(0).Infof("read dir %s attr %v: %v", dir.Path, request, err)
glog.V(2).Infof("read dir %s attr: %v, error: %v", dir.Path, dir.attributes, err)
return err
}
dir.attributes = entry.Attributes
if resp.Entry != nil {
dir.attributes = resp.Entry.Attributes
}
// dir.wfs.listDirectoryEntriesCache.Set(dir.Path, resp.Entry, dir.wfs.option.EntryCacheTtl)
return nil
})
if err != nil {
return err
}
// glog.V(1).Infof("dir %s: %v", dir.Path, attributes)
// glog.V(1).Infof("dir %s permission: %v", dir.Path, os.FileMode(attributes.FileMode))
glog.V(2).Infof("dir %s: %v perm: %v", dir.Path, dir.attributes, os.FileMode(dir.attributes.FileMode))
attr.Mode = os.FileMode(dir.attributes.FileMode) | os.ModeDir
@ -99,6 +70,16 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
return nil
}
func (dir *Dir) setRootDirAttributes(attr *fuse.Attr) {
attr.Uid = dir.wfs.option.MountUid
attr.Gid = dir.wfs.option.MountGid
attr.Mode = dir.wfs.option.MountMode
attr.Crtime = dir.wfs.option.MountCtime
attr.Ctime = dir.wfs.option.MountCtime
attr.Mtime = dir.wfs.option.MountMtime
attr.Atime = dir.wfs.option.MountMtime
}
func (dir *Dir) newFile(name string, entry *filer_pb.Entry) *File {
return &File{
Name: name,
@ -132,7 +113,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
glog.V(1).Infof("create: %v", request)
if request.Entry.IsDirectory {
if err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
if err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
if _, err := client.CreateEntry(ctx, request); err != nil {
glog.V(0).Infof("create %s/%s: %v", dir.Path, req.Name, err)
return fuse.EIO
@ -155,7 +136,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.CreateEntryRequest{
Directory: dir.Path,
@ -192,33 +173,18 @@ func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, err
func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fs.Node, err error) {
var entry *filer_pb.Entry
fullFilePath := path.Join(dir.Path, req.Name)
item := dir.wfs.listDirectoryEntriesCache.Get(path.Join(dir.Path, req.Name))
item := dir.wfs.listDirectoryEntriesCache.Get(fullFilePath)
if item != nil && !item.Expired() {
entry = item.Value().(*filer_pb.Entry)
}
if entry == nil {
err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.LookupDirectoryEntryRequest{
Directory: dir.Path,
Name: req.Name,
}
glog.V(4).Infof("lookup directory entry: %v", request)
resp, err := client.LookupDirectoryEntry(ctx, request)
entry, err = filer2.GetEntry(ctx, dir.wfs, fullFilePath)
if err != nil {
// glog.V(0).Infof("lookup %s/%s: %v", dir.Path, name, err)
return fuse.ENOENT
return nil, err
}
entry = resp.Entry
// dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, dir.wfs.option.EntryCacheTtl)
return nil
})
}
if entry != nil {
@ -243,7 +209,7 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
err = dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
paginationLimit := 1024
remaining := dir.wfs.option.DirListingLimit
@ -305,33 +271,14 @@ func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) error {
var entry *filer_pb.Entry
err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.LookupDirectoryEntryRequest{
Directory: dir.Path,
Name: req.Name,
}
glog.V(4).Infof("lookup to-be-removed entry: %v", request)
resp, err := client.LookupDirectoryEntry(ctx, request)
if err != nil {
// glog.V(0).Infof("lookup %s/%s: %v", dir.Path, name, err)
return fuse.ENOENT
}
entry = resp.Entry
return nil
})
entry, err := filer2.GetEntry(ctx, dir.wfs, path.Join(dir.Path, req.Name))
if err != nil {
return err
}
dir.wfs.deleteFileChunks(entry.Chunks)
dir.wfs.deleteFileChunks(ctx, entry.Chunks)
return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.DeleteEntryRequest{
Directory: dir.Path,
@ -355,7 +302,7 @@ func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) erro
func (dir *Dir) removeFolder(ctx context.Context, req *fuse.RemoveRequest) error {
return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.DeleteEntryRequest{
Directory: dir.Path,
@ -379,6 +326,10 @@ func (dir *Dir) removeFolder(ctx context.Context, req *fuse.RemoveRequest) error
func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
if dir.attributes == nil {
return nil
}
glog.V(3).Infof("%v dir setattr %+v, fh=%d", dir.Path, req, req.Handle)
if req.Valid.Mode() {
dir.attributes.FileMode = uint32(req.Mode)
@ -397,7 +348,7 @@ func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fus
}
parentDir, name := filer2.FullPath(dir.Path).DirAndName()
return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.UpdateEntryRequest{
Directory: parentDir,

2
weed/filesys/dir_link.go
View File

@ -35,7 +35,7 @@ func (dir *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node,
},
}
err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
if _, err := client.CreateEntry(ctx, request); err != nil {
glog.V(0).Infof("symlink %s/%s: %v", dir.Path, req.NewName, err)
return fuse.EIO

110
weed/filesys/dir_rename.go
View File

@ -2,118 +2,32 @@ package filesys
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/fuse"
"github.com/seaweedfs/fuse/fs"
"math"
"path/filepath"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)
func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {
newDir := newDirectory.(*Dir)
return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
// find existing entry
request := &filer_pb.LookupDirectoryEntryRequest{
Directory: dir.Path,
Name: req.OldName,
request := &filer_pb.AtomicRenameEntryRequest{
OldDirectory: dir.Path,
OldName: req.OldName,
NewDirectory: newDir.Path,
NewName: req.NewName,
}
glog.V(4).Infof("find existing directory entry: %v", request)
resp, err := client.LookupDirectoryEntry(ctx, request)
_, err := client.AtomicRenameEntry(ctx, request)
if err != nil {
glog.V(3).Infof("renaming find %s/%s: %v", dir.Path, req.OldName, err)
return fuse.ENOENT
}
entry := resp.Entry
glog.V(4).Infof("found existing directory entry resp: %+v", resp)
return moveEntry(ctx, client, dir.Path, entry, newDir.Path, req.NewName)
})
}
func moveEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, oldParent string, entry *filer_pb.Entry, newParent, newName string) error {
if entry.IsDirectory {
currentDirPath := filepath.Join(oldParent, entry.Name)
lastFileName := ""
includeLastFile := false
limit := math.MaxInt32
for limit > 0 {
request := &filer_pb.ListEntriesRequest{
Directory: currentDirPath,
StartFromFileName: lastFileName,
InclusiveStartFrom: includeLastFile,
Limit: 1024,
}
glog.V(4).Infof("read directory: %v", request)
resp, err := client.ListEntries(ctx, request)
if err != nil {
glog.V(0).Infof("list %s: %v", oldParent, err)
return fuse.EIO
}
if len(resp.Entries) == 0 {
break
}
for _, item := range resp.Entries {
lastFileName = item.Name
err := moveEntry(ctx, client, currentDirPath, item, filepath.Join(newParent, newName), item.Name)
if err != nil {
return err
}
limit--
}
if len(resp.Entries) < 1024 {
break
}
}
}
// add to new directory
{
request := &filer_pb.CreateEntryRequest{
Directory: newParent,
Entry: &filer_pb.Entry{
Name: newName,
IsDirectory: entry.IsDirectory,
Attributes: entry.Attributes,
Chunks: entry.Chunks,
},
}
glog.V(1).Infof("create new entry: %v", request)
if _, err := client.CreateEntry(ctx, request); err != nil {
glog.V(0).Infof("renaming create %s/%s: %v", newParent, newName, err)
return fuse.EIO
}
}
// delete old entry
{
request := &filer_pb.DeleteEntryRequest{
Directory: oldParent,
Name: entry.Name,
IsDeleteData: false,
}
glog.V(1).Infof("remove old entry: %v", request)
_, err := client.DeleteEntry(ctx, request)
if err != nil {
glog.V(0).Infof("renaming delete %s/%s: %v", oldParent, entry.Name, err)
return fuse.EIO
}
return fmt.Errorf("renaming %s/%s => %s/%s: %v", dir.Path, req.OldName, newDir.Path, req.NewName, err)
}
return nil
})
}

14
weed/filesys/dirty_page.go
View File

@ -4,13 +4,14 @@ import (
"bytes"
"context"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"sync"
"github.com/chrislusf/seaweedfs/weed/security"
)
type ContinuousDirtyPages struct {
@ -109,7 +110,7 @@ func (pages *ContinuousDirtyPages) flushAndSave(ctx context.Context, offset int6
// flush existing
if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil {
if chunk != nil {
glog.V(4).Infof("%s/%s flush existing [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
glog.V(4).Infof("%s/%s flush existing [%d,%d) to %s", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.FileId)
chunks = append(chunks, chunk)
}
} else {
@ -122,7 +123,7 @@ func (pages *ContinuousDirtyPages) flushAndSave(ctx context.Context, offset int6
// flush the new page
if chunk, err = pages.saveToStorage(ctx, data, offset); err == nil {
if chunk != nil {
glog.V(4).Infof("%s/%s flush big request [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
glog.V(4).Infof("%s/%s flush big request [%d,%d) to %s", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.FileId)
chunks = append(chunks, chunk)
}
} else {
@ -164,8 +165,9 @@ func (pages *ContinuousDirtyPages) saveExistingPagesToStorage(ctx context.Contex
func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte, offset int64) (*filer_pb.FileChunk, error) {
var fileId, host string
var auth security.EncodedJwt
if err := pages.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
if err := pages.f.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.AssignVolumeRequest{
Count: 1,
@ -181,7 +183,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte
return err
}
fileId, host = resp.FileId, resp.Url
fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth)
return nil
}); err != nil {
@ -190,7 +192,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte
fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
bufReader := bytes.NewReader(buf)
uploadResult, err := operation.Upload(fileUrl, pages.f.Name, bufReader, false, "application/octet-stream", nil, "")
uploadResult, err := operation.Upload(fileUrl, pages.f.Name, bufReader, false, "application/octet-stream", nil, auth)
if err != nil {
glog.V(0).Infof("upload data %v to %s: %v", pages.f.Name, fileUrl, err)
return nil, fmt.Errorf("upload data: %v", err)

14
weed/filesys/file.go
View File

@ -74,10 +74,6 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
return err
}
if file.isOpen {
return nil
}
glog.V(3).Infof("%v file setattr %+v, old:%+v", file.fullpath(), req, file.entry.Attributes)
if req.Valid.Size() {
@ -109,7 +105,11 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
file.entry.Attributes.Mtime = req.Mtime.Unix()
}
return file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
if file.isOpen {
return nil
}
return file.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.UpdateEntryRequest{
Directory: file.dir.Path,
@ -144,7 +144,7 @@ func (file *File) maybeLoadAttributes(ctx context.Context) error {
file.setEntry(entry)
// glog.V(1).Infof("file attr read cached %v attributes", file.Name)
} else {
err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
err := file.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.LookupDirectoryEntryRequest{
Name: file.Name,
@ -194,6 +194,8 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) {
newVisibles = t
}
glog.V(3).Infof("%s existing %d chunks adds %d more", file.fullpath(), len(file.entry.Chunks), len(chunks))
file.entry.Chunks = append(file.entry.Chunks, chunks...)
}

157
weed/filesys/filehandle.go
View File

@ -3,17 +3,16 @@ package filesys
import (
"context"
"fmt"
"mime"
"path"
"time"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/gabriel-vasile/mimetype"
"github.com/seaweedfs/fuse"
"github.com/seaweedfs/fuse/fs"
"net/http"
"strings"
"sync"
"time"
)
type FileHandle struct {
@ -65,75 +64,14 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus
chunkViews := filer2.ViewFromVisibleIntervals(fh.f.entryViewCache, req.Offset, req.Size)
var vids []string
for _, chunkView := range chunkViews {
vids = append(vids, volumeId(chunkView.FileId))
}
vid2Locations := make(map[string]*filer_pb.Locations)
err := fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
glog.V(4).Infof("read fh lookup volume id locations: %v", vids)
resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
VolumeIds: vids,
})
if err != nil {
return err
}
vid2Locations = resp.LocationsMap
return nil
})
if err != nil {
glog.V(4).Infof("%v/%v read fh lookup volume ids: %v", fh.f.dir.Path, fh.f.Name, err)
return fmt.Errorf("failed to lookup volume ids %v: %v", vids, err)
}
var totalRead int64
var wg sync.WaitGroup
for _, chunkView := range chunkViews {
wg.Add(1)
go func(chunkView *filer2.ChunkView) {
defer wg.Done()
glog.V(4).Infof("read fh reading chunk: %+v", chunkView)
locations := vid2Locations[volumeId(chunkView.FileId)]
if locations == nil || len(locations.Locations) == 0 {
glog.V(0).Infof("failed to locate %s", chunkView.FileId)
err = fmt.Errorf("failed to locate %s", chunkView.FileId)
return
}
var n int64
n, err = util.ReadUrl(
fmt.Sprintf("http://%s/%s", locations.Locations[0].Url, chunkView.FileId),
chunkView.Offset,
int(chunkView.Size),
buff[chunkView.LogicOffset-req.Offset:chunkView.LogicOffset-req.Offset+int64(chunkView.Size)],
!chunkView.IsFullChunk)
if err != nil {
glog.V(0).Infof("%v/%v read http://%s/%v %v bytes: %v", fh.f.dir.Path, fh.f.Name, locations.Locations[0].Url, chunkView.FileId, n, err)
err = fmt.Errorf("failed to read http://%s/%s: %v",
locations.Locations[0].Url, chunkView.FileId, err)
return
}
glog.V(4).Infof("read fh read %d bytes: %+v", n, chunkView)
totalRead += n
}(chunkView)
}
wg.Wait()
totalRead, err := filer2.ReadIntoBuffer(ctx, fh.f.wfs, fh.f.fullpath(), buff, chunkViews, req.Offset)
resp.Data = buff[:totalRead]
if err != nil {
glog.Errorf("file handle read %s: %v", fh.f.fullpath(), err)
}
return err
}
@ -153,7 +91,13 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
resp.Size = len(req.Data)
if req.Offset == 0 {
fh.contentType = http.DetectContentType(req.Data)
// detect mime type
var possibleExt string
fh.contentType, possibleExt = mimetype.Detect(req.Data)
if ext := path.Ext(fh.f.Name); ext != possibleExt {
fh.contentType = mime.TypeByExtension(ext)
}
fh.dirtyMetadata = true
}
@ -196,7 +140,7 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
return nil
}
return fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
return fh.f.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
if fh.f.entry.Attributes != nil {
fh.f.entry.Attributes.Mime = fh.contentType
@ -212,70 +156,25 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
Entry: fh.f.entry,
}
//glog.V(1).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.entry.Chunks))
//for i, chunk := range fh.f.entry.Chunks {
// glog.V(4).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
//}
glog.V(3).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.entry.Chunks))
for i, chunk := range fh.f.entry.Chunks {
glog.V(3).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
}
chunks, garbages := filer2.CompactFileChunks(fh.f.entry.Chunks)
fh.f.entry.Chunks = chunks
// fh.f.entryViewCache = nil
fh.f.wfs.deleteFileChunks(garbages)
if _, err := client.CreateEntry(ctx, request); err != nil {
glog.Errorf("update fh: %v", err)
return fmt.Errorf("update fh: %v", err)
}
fh.f.wfs.deleteFileChunks(ctx, garbages)
for i, chunk := range garbages {
glog.V(3).Infof("garbage %s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
}
return nil
})
}
func deleteFileIds(ctx context.Context, client filer_pb.SeaweedFilerClient, fileIds []string) error {
var vids []string
for _, fileId := range fileIds {
vids = append(vids, volumeId(fileId))
}
lookupFunc := func(vids []string) (map[string]operation.LookupResult, error) {
m := make(map[string]operation.LookupResult)
glog.V(4).Infof("remove file lookup volume id locations: %v", vids)
resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
VolumeIds: vids,
})
if err != nil {
return m, err
}
for _, vid := range vids {
lr := operation.LookupResult{
VolumeId: vid,
Locations: nil,
}
locations := resp.LocationsMap[vid]
for _, loc := range locations.Locations {
lr.Locations = append(lr.Locations, operation.Location{
Url: loc.Url,
PublicUrl: loc.PublicUrl,
})
}
m[vid] = lr
}
return m, err
}
_, err := operation.DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
return err
}
func volumeId(fileId string) string {
lastCommaIndex := strings.LastIndex(fileId, ",")
if lastCommaIndex > 0 {
return fileId[:lastCommaIndex]
}
return fileId
}

16
weed/filesys/wfs.go
View File

@ -19,6 +19,7 @@ import (
type Option struct {
FilerGrpcAddress string
GrpcDialOption grpc.DialOption
FilerMountRootPath string
Collection string
Replication string
@ -31,6 +32,8 @@ type Option struct {
MountUid uint32
MountGid uint32
MountMode os.FileMode
MountCtime time.Time
MountMtime time.Time
}
var _ = fs.FS(&WFS{})
@ -46,8 +49,6 @@ type WFS struct {
pathToHandleLock sync.Mutex
bufPool sync.Pool
fileIdsDeletionChan chan []string
stats statsCache
}
type statsCache struct {
@ -65,11 +66,8 @@ func NewSeaweedFileSystem(option *Option) *WFS {
return make([]byte, option.ChunkSizeLimit)
},
},
fileIdsDeletionChan: make(chan []string, 32),
}
go wfs.loopProcessingDeletion()
return wfs
}
@ -77,12 +75,12 @@ func (wfs *WFS) Root() (fs.Node, error) {
return &Dir{Path: wfs.option.FilerMountRootPath, wfs: wfs}, nil
}
func (wfs *WFS) withFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
func (wfs *WFS) WithFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error {
return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
return fn(client)
}, wfs.option.FilerGrpcAddress)
}, wfs.option.FilerGrpcAddress, wfs.option.GrpcDialOption)
}
@ -137,7 +135,7 @@ func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.
if wfs.stats.lastChecked < time.Now().Unix()-20 {
err := wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
err := wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.StatisticsRequest{
Collection: wfs.option.Collection,

87
weed/filesys/wfs_deletion.go
View File

@ -2,57 +2,68 @@ package filesys
import (
"context"
"time"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"google.golang.org/grpc"
)
func (wfs *WFS) loopProcessingDeletion() {
ticker := time.NewTicker(2 * time.Second)
wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
var fileIds []string
for {
select {
case fids := <-wfs.fileIdsDeletionChan:
fileIds = append(fileIds, fids...)
if len(fileIds) >= 1024 {
glog.V(1).Infof("deleting fileIds len=%d", len(fileIds))
deleteFileIds(context.Background(), client, fileIds)
fileIds = fileIds[:0]
}
case <-ticker.C:
if len(fileIds) > 0 {
glog.V(1).Infof("timed deletion fileIds len=%d", len(fileIds))
deleteFileIds(context.Background(), client, fileIds)
fileIds = fileIds[:0]
}
}
}
})
}
func (wfs *WFS) deleteFileChunks(chunks []*filer_pb.FileChunk) {
func (wfs *WFS) deleteFileChunks(ctx context.Context, chunks []*filer_pb.FileChunk) {
if len(chunks) == 0 {
return
}
var fileIds []string
for _, chunk := range chunks {
fileIds = append(fileIds, chunk.FileId)
fileIds = append(fileIds, chunk.GetFileIdString())
}
var async = false
if async {
wfs.fileIdsDeletionChan <- fileIds
return
}
wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
deleteFileIds(context.Background(), client, fileIds)
wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
deleteFileIds(ctx, wfs.option.GrpcDialOption, client, fileIds)
return nil
})
}
func deleteFileIds(ctx context.Context, grpcDialOption grpc.DialOption, client filer_pb.SeaweedFilerClient, fileIds []string) error {
var vids []string
for _, fileId := range fileIds {
vids = append(vids, filer2.VolumeId(fileId))
}
lookupFunc := func(vids []string) (map[string]operation.LookupResult, error) {
m := make(map[string]operation.LookupResult)
glog.V(4).Infof("remove file lookup volume id locations: %v", vids)
resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
VolumeIds: vids,
})
if err != nil {
return m, err
}
for _, vid := range vids {
lr := operation.LookupResult{
VolumeId: vid,
Locations: nil,
}
locations := resp.LocationsMap[vid]
for _, loc := range locations.Locations {
lr.Locations = append(lr.Locations, operation.Location{
Url: loc.Url,
PublicUrl: loc.PublicUrl,
})
}
m[vid] = lr
}
return m, err
}
_, err := operation.DeleteFilesWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc)
return err
}

190
weed/glide.lock generated
View File

@ -1,190 +0,0 @@
hash: 2e3a065472829938d25e879451b6d1aa43e55270e1166a9c044803ef8a3b9eb1
updated: 2018-06-28T22:01:35.910567-07:00
imports:
- name: github.com/seaweedfs/fuse
version: 65cc252bf6691cb3c7014bcb2c8dc29de91e3a7e
subpackages:
- fs
- fuseutil
- name: github.com/boltdb/bolt
version: 2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8
- name: github.com/chrislusf/raft
version: 5f7ddd8f479583daf05879d3d3b174aa202c8fb7
subpackages:
- protobuf
- name: github.com/dgrijalva/jwt-go
version: 06ea1031745cb8b3dab3f6a236daf2b0aa468b7e
- name: github.com/disintegration/imaging
version: bbcee2f5c9d5e94ca42c8b50ec847fec64a6c134
- name: github.com/fsnotify/fsnotify
version: c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9
- name: github.com/go-redis/redis
version: 83fb42932f6145ce52df09860384a4653d2d332a
subpackages:
- internal
- internal/consistenthash
- internal/hashtag
- internal/pool
- internal/proto
- internal/singleflight
- internal/util
- name: github.com/go-sql-driver/mysql
version: d523deb1b23d913de5bdada721a6071e71283618
- name: github.com/gocql/gocql
version: e06f8c1bcd787e6bf0608288b314522f08cc7848
subpackages:
- internal/lru
- internal/murmur
- internal/streams
- name: github.com/gogo/protobuf
version: 30cf7ac33676b5786e78c746683f0d4cd64fa75b
subpackages:
- proto
- name: github.com/golang/protobuf
version: b4deda0973fb4c70b50d226b1af49f3da59f5265
subpackages:
- proto
- protoc-gen-go/descriptor
- ptypes
- ptypes/any
- ptypes/duration
- ptypes/timestamp
- name: github.com/golang/snappy
version: 2e65f85255dbc3072edf28d6b5b8efc472979f5a
- name: github.com/google/btree
version: e89373fe6b4a7413d7acd6da1725b83ef713e6e4
- name: github.com/gorilla/context
version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42
- name: github.com/gorilla/mux
version: e3702bed27f0d39777b0b37b664b6280e8ef8fbf
- name: github.com/hailocab/go-hostpool
version: e80d13ce29ede4452c43dea11e79b9bc8a15b478
- name: github.com/hashicorp/hcl
version: ef8a98b0bbce4a65b5aa4c368430a80ddc533168
subpackages:
- hcl/ast
- hcl/parser
- hcl/printer
- hcl/scanner
- hcl/strconv
- hcl/token
- json/parser
- json/scanner
- json/token
- name: github.com/karlseguin/ccache
version: b425c9ca005a2050ebe723f6a0cddcb907354ab7
- name: github.com/klauspost/crc32
version: cb6bfca970f6908083f26f39a79009d608efd5cd
- name: github.com/lib/pq
version: 90697d60dd844d5ef6ff15135d0203f65d2f53b8
subpackages:
- oid
- name: github.com/magiconair/properties
version: c2353362d570a7bfa228149c62842019201cfb71
- name: github.com/mitchellh/mapstructure
version: bb74f1db0675b241733089d5a1faa5dd8b0ef57b
- name: github.com/pelletier/go-toml
version: c01d1270ff3e442a8a57cddc1c92dc1138598194
- name: github.com/rwcarlsen/goexif
version: 8d986c03457a2057c7b0fb0a48113f7dd48f9619
subpackages:
- exif
- tiff
- name: github.com/soheilhy/cmux
version: e09e9389d85d8492d313d73d1469c029e710623f
- name: github.com/spf13/afero
version: 787d034dfe70e44075ccc060d346146ef53270ad
subpackages:
- mem
- name: github.com/spf13/cast
version: 8965335b8c7107321228e3e3702cab9832751bac
- name: github.com/spf13/jwalterweatherman
version: 7c0cea34c8ece3fbeb2b27ab9b59511d360fb394
- name: github.com/spf13/pflag
version: 3ebe029320b2676d667ae88da602a5f854788a8a
- name: github.com/spf13/viper
version: 15738813a09db5c8e5b60a19d67d3f9bd38da3a4
- name: github.com/syndtr/goleveldb
version: 0d5a0ceb10cf9ab89fdd744cc8c50a83134f6697
subpackages:
- leveldb
- leveldb/cache
- leveldb/comparer
- leveldb/errors
- leveldb/filter
- leveldb/iterator
- leveldb/journal
- leveldb/memdb
- leveldb/opt
- leveldb/storage
- leveldb/table
- leveldb/util
- name: golang.org/x/image
version: cc896f830cedae125428bc9fe1b0362aa91b3fb1
subpackages:
- bmp
- tiff
- tiff/lzw
- name: golang.org/x/net
version: 4cb1c02c05b0e749b0365f61ae859a8e0cfceed9
subpackages:
- context
- http/httpguts
- http2
- http2/hpack
- idna
- internal/timeseries
- trace
- name: golang.org/x/sys
version: 7138fd3d9dc8335c567ca206f4333fb75eb05d56
subpackages:
- unix
- name: golang.org/x/text
version: 5cec4b58c438bd98288aeb248bab2c1840713d21
subpackages:
- secure/bidirule
- transform
- unicode/bidi
- unicode/norm
- name: google.golang.org/appengine
version: b1f26356af11148e710935ed1ac8a7f5702c7612
subpackages:
- cloudsql
- name: google.golang.org/genproto
version: ff3583edef7de132f219f0efc00e097cabcc0ec0
subpackages:
- googleapis/rpc/status
- name: google.golang.org/grpc
version: 168a6198bcb0ef175f7dacec0b8691fc141dc9b8
subpackages:
- balancer
- balancer/base
- balancer/roundrobin
- codes
- connectivity
- credentials
- encoding
- encoding/proto
- grpclog
- internal
- internal/backoff
- internal/channelz
- internal/grpcrand
- keepalive
- metadata
- naming
- peer
- reflection
- reflection/grpc_reflection_v1alpha
- resolver
- resolver/dns
- resolver/passthrough
- stats
- status
- tap
- transport
- name: gopkg.in/inf.v0
version: d2d2541c53f18d2a059457998ce2876cc8e67cbf
- name: gopkg.in/yaml.v2
version: 5420a8b6744d3b0345ab293f6fcba19c978f1183
testImports: []

101
weed/glide.yaml
View File

@ -1,44 +1,117 @@
package: github.com/chrislusf/seaweedfs/weed
import:
- package: github.com/seaweedfs/fuse
- package: cloud.google.com/go
version: ^0.40.0
subpackages:
- fs
- package: github.com/boltdb/bolt
version: ^1.3.1
- pubsub
- storage
- package: github.com/Azure/azure-storage-blob-go
version: ^0.7.0
subpackages:
- azblob
- package: github.com/Shopify/sarama
version: ^1.22.1
- package: github.com/aws/aws-sdk-go
version: ^1.20.12
subpackages:
- aws
- aws/awserr
- aws/credentials
- aws/session
- service/s3
- service/s3/s3iface
- service/sqs
- package: github.com/chrislusf/raft
subpackages:
- protobuf
- package: github.com/dgrijalva/jwt-go
version: ^3.2.0
- package: github.com/disintegration/imaging
version: ^1.4.1
version: ^1.6.0
- package: github.com/dustin/go-humanize
version: ^1.0.0
- package: github.com/gabriel-vasile/mimetype
version: ^0.3.14
- package: github.com/go-redis/redis
version: ^6.10.2
version: ^6.15.3
- package: github.com/go-sql-driver/mysql
version: ^1.3.0
version: ^1.4.1
- package: github.com/gocql/gocql
- package: github.com/golang/protobuf
version: ^1.0.0
version: ^1.3.1
subpackages:
- proto
- package: github.com/google/btree
version: ^1.0.0
- package: github.com/gorilla/mux
version: ^1.6.1
version: ^1.7.3
- package: github.com/jacobsa/daemonize
- package: github.com/kardianos/osext
- package: github.com/karlseguin/ccache
version: ^2.0.3
- package: github.com/klauspost/crc32
version: ^1.1.0
version: ^1.2.0
- package: github.com/klauspost/reedsolomon
version: ^1.9.2
- package: github.com/kurin/blazer
version: ^0.5.3
subpackages:
- b2
- package: github.com/lib/pq
version: ^1.1.1
- package: github.com/peterh/liner
version: ^1.1.0
- package: github.com/prometheus/client_golang
version: ^1.0.0
subpackages:
- prometheus
- prometheus/push
- package: github.com/rakyll/statik
version: ^0.1.6
subpackages:
- fs
- package: github.com/rwcarlsen/goexif
subpackages:
- exif
- package: github.com/soheilhy/cmux
version: ^0.1.4
- package: github.com/satori/go.uuid
version: ^1.2.0
- package: github.com/seaweedfs/fuse
subpackages:
- fs
- package: github.com/spf13/viper
version: ^1.4.0
- package: github.com/syndtr/goleveldb
version: ^1.0.0
subpackages:
- leveldb
- leveldb/opt
- leveldb/util
- package: github.com/willf/bloom
version: ^2.0.3
- package: gocloud.dev
version: ^0.15.0
subpackages:
- pubsub
- pubsub/awssnssqs
- pubsub/azuresb
- pubsub/gcppubsub
- pubsub/natspubsub
- pubsub/rabbitpubsub
- package: golang.org/x/net
subpackages:
- context
- package: google.golang.org/grpc
version: ^1.11.3
- webdav
- package: golang.org/x/tools
subpackages:
- godoc/util
- package: google.golang.org/api
version: ^0.7.0
subpackages:
- option
- package: google.golang.org/grpc
version: ^1.21.1
subpackages:
- credentials
- keepalive
- peer
- reflection

71
weed/notification/gocdk_pub_sub/gocdk_pub_sub.go Normal file
View File

@ -0,0 +1,71 @@
// Package gocdk_pub_sub supports the Go CDK (Cloud Development Kit) PubSub API,
// which in turn supports many providers, including Amazon SNS/SQS, Azure Service Bus,
// Google Cloud PubSub, and RabbitMQ.
//
// In the config, select a provider and topic using a URL. See
// https://godoc.org/gocloud.dev/pubsub and its sub-packages for details.
//
// The Go CDK PubSub API does not support administrative operations like topic
// creation. Create the topic using a UI, CLI or provider-specific API before running
// weed.
//
// The Go CDK obtains credentials via environment variables and other
// provider-specific default mechanisms. See the provider's documentation for
// details.
package gocdk_pub_sub
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/notification"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/golang/protobuf/proto"
"gocloud.dev/pubsub"
_ "gocloud.dev/pubsub/awssnssqs"
_ "gocloud.dev/pubsub/azuresb"
_ "gocloud.dev/pubsub/gcppubsub"
_ "gocloud.dev/pubsub/natspubsub"
_ "gocloud.dev/pubsub/rabbitpubsub"
)
func init() {
notification.MessageQueues = append(notification.MessageQueues, &GoCDKPubSub{})
}
type GoCDKPubSub struct {
topicURL string
topic *pubsub.Topic
}
func (k *GoCDKPubSub) GetName() string {
return "gocdk_pub_sub"
}
func (k *GoCDKPubSub) Initialize(config util.Configuration) error {
k.topicURL = config.GetString("topic_url")
glog.V(0).Infof("notification.gocdk_pub_sub.topic_url: %v", k.topicURL)
topic, err := pubsub.OpenTopic(context.Background(), k.topicURL)
if err != nil {
glog.Fatalf("Failed to open topic: %v", err)
}
k.topic = topic
return nil
}
func (k *GoCDKPubSub) SendMessage(key string, message proto.Message) error {
bytes, err := proto.Marshal(message)
if err != nil {
return err
}
ctx := context.Background()
err = k.topic.Send(ctx, &pubsub.Message{
Body: bytes,
Metadata: map[string]string{"key": key},
})
if err != nil {
return fmt.Errorf("send message via Go CDK pubsub %s: %v", k.topicURL, err)
}
return nil
}

30
weed/operation/assign_file_id.go
View File

@ -3,9 +3,11 @@ package operation
import (
"context"
"fmt"
"time"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"google.golang.org/grpc"
"strings"
)
type VolumeAssignRequest struct {
@ -24,9 +26,10 @@ type AssignResult struct {
PublicUrl string `json:"publicUrl,omitempty"`
Count uint64 `json:"count,omitempty"`
Error string `json:"error,omitempty"`
Auth security.EncodedJwt `json:"auth,omitempty"`
}
func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeRequests ...*VolumeAssignRequest) (*AssignResult, error) {
func Assign(server string, grpcDialOption grpc.DialOption, primaryRequest *VolumeAssignRequest, alternativeRequests ...*VolumeAssignRequest) (*AssignResult, error) {
var requests []*VolumeAssignRequest
requests = append(requests, primaryRequest)
@ -40,9 +43,7 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque
continue
}
lastError = withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
defer cancel()
lastError = WithMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
req := &master_pb.AssignRequest{
Count: primaryRequest.Count,
@ -53,7 +54,7 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque
Rack: primaryRequest.Rack,
DataNode: primaryRequest.DataNode,
}
resp, grpcErr := masterClient.Assign(ctx, req)
resp, grpcErr := masterClient.Assign(context.Background(), req)
if grpcErr != nil {
return grpcErr
}
@ -63,6 +64,7 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque
ret.Url = resp.Url
ret.PublicUrl = resp.PublicUrl
ret.Error = resp.Error
ret.Auth = security.EncodedJwt(resp.Auth)
return nil
@ -81,3 +83,17 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque
return ret, lastError
}
func LookupJwt(master string, fileId string) security.EncodedJwt {
tokenStr := ""
if h, e := util.Head(fmt.Sprintf("http://%s/dir/lookup?fileId=%s", master, fileId)); e == nil {
bearer := h.Get("Authorization")
if len(bearer) > 7 && strings.ToUpper(bearer[0:6]) == "BEARER" {
tokenStr = bearer[7:]
}
}
return security.EncodedJwt(tokenStr)
}

14
weed/operation/chunked_file.go
View File

@ -5,9 +5,12 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"sort"
"google.golang.org/grpc"
"sync"
"github.com/chrislusf/seaweedfs/weed/glog"
@ -53,7 +56,7 @@ func (s ChunkList) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func LoadChunkManifest(buffer []byte, isGzipped bool) (*ChunkManifest, error) {
if isGzipped {
var err error
if buffer, err = UnGzipData(buffer); err != nil {
if buffer, err = util.UnGzipData(buffer); err != nil {
return nil, err
}
}
@ -69,12 +72,12 @@ func (cm *ChunkManifest) Marshal() ([]byte, error) {
return json.Marshal(cm)
}
func (cm *ChunkManifest) DeleteChunks(master string) error {
func (cm *ChunkManifest) DeleteChunks(master string, grpcDialOption grpc.DialOption) error {
var fileIds []string
for _, ci := range cm.Chunks {
fileIds = append(fileIds, ci.Fid)
}
results, err := DeleteFiles(master, fileIds)
results, err := DeleteFiles(master, grpcDialOption, fileIds)
if err != nil {
glog.V(0).Infof("delete %+v: %v", fileIds, err)
return fmt.Errorf("chunk delete: %v", err)
@ -102,7 +105,10 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64) (written int64,
if err != nil {
return written, err
}
defer resp.Body.Close()
defer func() {
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
}()
switch resp.StatusCode {
case http.StatusRequestedRangeNotSatisfiable:

1
weed/operation/data_struts.go
View File

@ -2,6 +2,5 @@ package operation
type JoinResult struct {
VolumeSizeLimit uint64 `json:"VolumeSizeLimit,omitempty"`
SecretKey string `json:"secretKey,omitempty"`
Error string `json:"error,omitempty"`
}

37
weed/operation/delete_content.go
View File

@ -4,12 +4,12 @@ import (
"context"
"errors"
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"google.golang.org/grpc"
"net/http"
"strings"
"sync"
"time"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
)
type DeleteResult struct {
@ -28,17 +28,17 @@ func ParseFileId(fid string) (vid string, key_cookie string, err error) {
}
// DeleteFiles batch deletes a list of fileIds
func DeleteFiles(master string, fileIds []string) ([]*volume_server_pb.DeleteResult, error) {
func DeleteFiles(master string, grpcDialOption grpc.DialOption, fileIds []string) ([]*volume_server_pb.DeleteResult, error) {
lookupFunc := func(vids []string) (map[string]LookupResult, error) {
return LookupVolumeIds(master, vids)
return LookupVolumeIds(master, grpcDialOption, vids)
}
return DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
return DeleteFilesWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc)
}
func DeleteFilesWithLookupVolumeId(fileIds []string, lookupFunc func(vid []string) (map[string]LookupResult, error)) ([]*volume_server_pb.DeleteResult, error) {
func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func(vid []string) (map[string]LookupResult, error)) ([]*volume_server_pb.DeleteResult, error) {
var ret []*volume_server_pb.DeleteResult
@ -48,7 +48,7 @@ func DeleteFilesWithLookupVolumeId(fileIds []string, lookupFunc func(vid []strin
vid, _, err := ParseFileId(fileId)
if err != nil {
ret = append(ret, &volume_server_pb.DeleteResult{
FileId: vid,
FileId: fileId,
Status: http.StatusBadRequest,
Error: err.Error()},
)
@ -85,38 +85,43 @@ func DeleteFilesWithLookupVolumeId(fileIds []string, lookupFunc func(vid []strin
}
}
resultChan := make(chan []*volume_server_pb.DeleteResult, len(server_to_fileIds))
var wg sync.WaitGroup
for server, fidList := range server_to_fileIds {
wg.Add(1)
go func(server string, fidList []string) {
defer wg.Done()
if deleteResults, deleteErr := DeleteFilesAtOneVolumeServer(server, fidList); deleteErr != nil {
if deleteResults, deleteErr := DeleteFilesAtOneVolumeServer(server, grpcDialOption, fidList); deleteErr != nil {
err = deleteErr
} else {
ret = append(ret, deleteResults...)
resultChan <- deleteResults
}
}(server, fidList)
}
wg.Wait()
close(resultChan)
for result := range resultChan {
ret = append(ret, result...)
}
glog.V(0).Infof("deleted %d items", len(ret))
return ret, err
}
// DeleteFilesAtOneVolumeServer deletes a list of files that is on one volume server via gRpc
func DeleteFilesAtOneVolumeServer(volumeServer string, fileIds []string) (ret []*volume_server_pb.DeleteResult, err error) {
func DeleteFilesAtOneVolumeServer(volumeServer string, grpcDialOption grpc.DialOption, fileIds []string) (ret []*volume_server_pb.DeleteResult, err error) {
err = WithVolumeServerClient(volumeServer, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
defer cancel()
err = WithVolumeServerClient(volumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
req := &volume_server_pb.BatchDeleteRequest{
FileIds: fileIds,
}
resp, err := volumeServerClient.BatchDelete(ctx, req)
resp, err := volumeServerClient.BatchDelete(context.Background(), req)
// fmt.Printf("deleted %v %v: %v\n", fileIds, err, resp)

30
weed/operation/grpc_client.go
View File

@ -1,34 +1,30 @@
package operation
import (
"context"
"fmt"
"strconv"
"strings"
"sync"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"google.golang.org/grpc"
"strconv"
"strings"
)
var (
grpcClients = make(map[string]*grpc.ClientConn)
grpcClientsLock sync.Mutex
)
func WithVolumeServerClient(volumeServer string, grpcDialOption grpc.DialOption, fn func(volume_server_pb.VolumeServerClient) error) error {
func WithVolumeServerClient(volumeServer string, fn func(volume_server_pb.VolumeServerClient) error) error {
ctx := context.Background()
grpcAddress, err := toVolumeServerGrpcAddress(volumeServer)
if err != nil {
return err
}
return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
client := volume_server_pb.NewVolumeServerClient(grpcConnection)
return fn(client)
}, grpcAddress)
}, grpcAddress, grpcDialOption)
}
@ -42,16 +38,18 @@ func toVolumeServerGrpcAddress(volumeServer string) (grpcAddress string, err err
return fmt.Sprintf("%s:%d", volumeServer[0:sepIndex], port+10000), nil
}
func withMasterServerClient(masterServer string, fn func(masterClient master_pb.SeaweedClient) error) error {
func WithMasterServerClient(masterServer string, grpcDialOption grpc.DialOption, fn func(masterClient master_pb.SeaweedClient) error) error {
masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(masterServer, 0)
ctx := context.Background()
masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(masterServer)
if parseErr != nil {
return fmt.Errorf("failed to parse master grpc %v", masterServer)
return fmt.Errorf("failed to parse master grpc %v: %v", masterServer, parseErr)
}
return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
client := master_pb.NewSeaweedClient(grpcConnection)
return fn(client)
}, masterGrpcAddress)
}, masterGrpcAddress, grpcDialOption)
}

32
weed/operation/list_masters.go
View File

@ -1,32 +0,0 @@
package operation
import (
"encoding/json"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/util"
)
type ClusterStatusResult struct {
IsLeader bool `json:"IsLeader,omitempty"`
Leader string `json:"Leader,omitempty"`
Peers []string `json:"Peers,omitempty"`
}
func ListMasters(server string) (leader string, peers []string, err error) {
jsonBlob, err := util.Get("http://" + server + "/cluster/status")
glog.V(2).Info("list masters result :", string(jsonBlob))
if err != nil {
return "", nil, err
}
var ret ClusterStatusResult
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return "", nil, err
}
peers = ret.Peers
if ret.IsLeader {
peers = append(peers, ret.Leader)
}
return ret.Leader, peers, nil
}

9
weed/operation/lookup.go
View File

@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"google.golang.org/grpc"
"math/rand"
"net/url"
"strings"
@ -78,7 +79,7 @@ func LookupFileId(server string, fileId string) (fullUrl string, err error) {
}
// LookupVolumeIds find volume locations by cache and actual lookup
func LookupVolumeIds(server string, vids []string) (map[string]LookupResult, error) {
func LookupVolumeIds(server string, grpcDialOption grpc.DialOption, vids []string) (map[string]LookupResult, error) {
ret := make(map[string]LookupResult)
var unknown_vids []string
@ -98,14 +99,12 @@ func LookupVolumeIds(server string, vids []string) (map[string]LookupResult, err
//only query unknown_vids
err := withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
defer cancel()
err := WithMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
req := &master_pb.LookupVolumeRequest{
VolumeIds: unknown_vids,
}
resp, grpcErr := masterClient.LookupVolume(ctx, req)
resp, grpcErr := masterClient.LookupVolume(context.Background(), req)
if grpcErr != nil {
return grpcErr
}

10
weed/operation/stats.go
View File

@ -2,18 +2,16 @@ package operation
import (
"context"
"time"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
)
func Statistics(server string, req *master_pb.StatisticsRequest) (resp *master_pb.StatisticsResponse, err error) {
func Statistics(server string, grpcDialOption grpc.DialOption, req *master_pb.StatisticsRequest) (resp *master_pb.StatisticsResponse, err error) {
err = withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
defer cancel()
err = WithMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
grpcResponse, grpcErr := masterClient.Statistics(ctx, req)
grpcResponse, grpcErr := masterClient.Statistics(context.Background(), req)
if grpcErr != nil {
return grpcErr
}

26
weed/operation/submit.go
View File

@ -2,6 +2,7 @@ package operation
import (
"bytes"
"google.golang.org/grpc"
"io"
"mime"
"net/url"
@ -36,10 +37,8 @@ type SubmitResult struct {
Error string `json:"error,omitempty"`
}
func SubmitFiles(master string, files []FilePart,
replication string, collection string, dataCenter string, ttl string, maxMB int,
secret security.Secret,
) ([]SubmitResult, error) {
func SubmitFiles(master string, grpcDialOption grpc.DialOption, files []FilePart,
replication string, collection string, dataCenter string, ttl string, maxMB int) ([]SubmitResult, error) {
results := make([]SubmitResult, len(files))
for index, file := range files {
results[index].FileName = file.FileName
@ -51,7 +50,7 @@ func SubmitFiles(master string, files []FilePart,
DataCenter: dataCenter,
Ttl: ttl,
}
ret, err := Assign(master, ar)
ret, err := Assign(master, grpcDialOption, ar)
if err != nil {
for index, _ := range files {
results[index].Error = err.Error()
@ -67,7 +66,7 @@ func SubmitFiles(master string, files []FilePart,
file.Replication = replication
file.Collection = collection
file.DataCenter = dataCenter
results[index].Size, err = file.Upload(maxMB, master, secret)
results[index].Size, err = file.Upload(maxMB, master, ret.Auth, grpcDialOption)
if err != nil {
results[index].Error = err.Error()
}
@ -110,8 +109,7 @@ func newFilePart(fullPathFilename string) (ret FilePart, err error) {
return ret, nil
}
func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (retSize uint32, err error) {
jwt := security.GenJwt(secret, fi.Fid)
func (fi FilePart) Upload(maxMB int, master string, jwt security.EncodedJwt, grpcDialOption grpc.DialOption) (retSize uint32, err error) {
fileUrl := "http://" + fi.Server + "/" + fi.Fid
if fi.ModTime != 0 {
fileUrl += "?ts=" + strconv.Itoa(int(fi.ModTime))
@ -139,7 +137,7 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
Collection: fi.Collection,
Ttl: fi.Ttl,
}
ret, err = Assign(master, ar)
ret, err = Assign(master, grpcDialOption, ar)
if err != nil {
return
}
@ -152,10 +150,10 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
Collection: fi.Collection,
Ttl: fi.Ttl,
}
ret, err = Assign(master, ar)
ret, err = Assign(master, grpcDialOption, ar)
if err != nil {
// delete all uploaded chunks
cm.DeleteChunks(master)
cm.DeleteChunks(master, grpcDialOption)
return
}
id = ret.Fid
@ -170,10 +168,10 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
baseName+"-"+strconv.FormatInt(i+1, 10),
io.LimitReader(fi.Reader, chunkSize),
master, fileUrl,
jwt)
ret.Auth)
if e != nil {
// delete all uploaded chunks
cm.DeleteChunks(master)
cm.DeleteChunks(master, grpcDialOption)
return 0, e
}
cm.Chunks = append(cm.Chunks,
@ -188,7 +186,7 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
err = upload_chunked_file_manifest(fileUrl, &cm, jwt)
if err != nil {
// delete all uploaded chunks
cm.DeleteChunks(master)
cm.DeleteChunks(master, grpcDialOption)
}
} else {
ret, e := Upload(fileUrl, baseName, fi.Reader, false, fi.MimeType, nil, jwt)

54
weed/operation/sync_volume.go
View File

@ -2,63 +2,19 @@ package operation
import (
"context"
"fmt"
"io"
"time"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
"google.golang.org/grpc"
)
func GetVolumeSyncStatus(server string, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) {
func GetVolumeSyncStatus(server string, grpcDialOption grpc.DialOption, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) {
WithVolumeServerClient(server, func(client volume_server_pb.VolumeServerClient) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
defer cancel()
WithVolumeServerClient(server, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
resp, err = client.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{
VolumdId: vid,
resp, err = client.VolumeSyncStatus(context.Background(), &volume_server_pb.VolumeSyncStatusRequest{
VolumeId: vid,
})
return nil
})
return
}
func GetVolumeIdxEntries(server string, vid uint32, eachEntryFn func(key NeedleId, offset Offset, size uint32)) error {
return WithVolumeServerClient(server, func(client volume_server_pb.VolumeServerClient) error {
stream, err := client.VolumeSyncIndex(context.Background(), &volume_server_pb.VolumeSyncIndexRequest{
VolumdId: vid,
})
if err != nil {
return err
}
var indexFileContent []byte
for {
resp, err := stream.Recv()
if err == io.EOF {
break
}
if err != nil {
return fmt.Errorf("read index entries: %v", err)
}
indexFileContent = append(indexFileContent, resp.IndexFileContent...)
}
dataSize := len(indexFileContent)
for idx := 0; idx+NeedleEntrySize <= dataSize; idx += NeedleEntrySize {
line := indexFileContent[idx : idx+NeedleEntrySize]
key := BytesToNeedleId(line[:NeedleIdSize])
offset := BytesToOffset(line[NeedleIdSize : NeedleIdSize+OffsetSize])
size := util.BytesToUint32(line[NeedleIdSize+OffsetSize : NeedleIdSize+OffsetSize+SizeSize])
eachEntryFn(key, offset, size)
}
return nil
})
}

82
weed/operation/tail_volume.go Normal file
View File

@ -0,0 +1,82 @@
package operation
import (
"context"
"fmt"
"io"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"google.golang.org/grpc"
)
func TailVolume(master string, grpcDialOption grpc.DialOption, vid needle.VolumeId, sinceNs uint64, timeoutSeconds int, fn func(n *needle.Needle) error) error {
// find volume location, replication, ttl info
lookup, err := Lookup(master, vid.String())
if err != nil {
return fmt.Errorf("look up volume %d: %v", vid, err)
}
if len(lookup.Locations) == 0 {
return fmt.Errorf("unable to locate volume %d", vid)
}
volumeServer := lookup.Locations[0].Url
return TailVolumeFromSource(volumeServer, grpcDialOption, vid, sinceNs, timeoutSeconds, fn)
}
func TailVolumeFromSource(volumeServer string, grpcDialOption grpc.DialOption, vid needle.VolumeId, sinceNs uint64, idleTimeoutSeconds int, fn func(n *needle.Needle) error) error {
return WithVolumeServerClient(volumeServer, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
stream, err := client.VolumeTailSender(context.Background(), &volume_server_pb.VolumeTailSenderRequest{
VolumeId: uint32(vid),
SinceNs: sinceNs,
IdleTimeoutSeconds: uint32(idleTimeoutSeconds),
})
if err != nil {
return err
}
for {
resp, recvErr := stream.Recv()
if recvErr != nil {
if recvErr == io.EOF {
break
} else {
return recvErr
}
}
needleHeader := resp.NeedleHeader
needleBody := resp.NeedleBody
if len(needleHeader) == 0 {
continue
}
for !resp.IsLastChunk {
resp, recvErr = stream.Recv()
if recvErr != nil {
if recvErr == io.EOF {
break
} else {
return recvErr
}
}
needleBody = append(needleBody, resp.NeedleBody...)
}
n := new(needle.Needle)
n.ParseNeedleHeader(needleHeader)
n.ReadNeedleBodyBytes(needleBody, needle.CurrentVersion)
err = fn(n)
if err != nil {
return err
}
}
return nil
})
}

57
weed/operation/upload_content.go
View File

@ -2,6 +2,8 @@ package operation
import (
"bytes"
"compress/flate"
"compress/gzip"
"encoding/json"
"errors"
"fmt"
@ -16,6 +18,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
)
type UploadResult struct {
@ -37,13 +40,43 @@ func init() {
var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
// Upload sends a POST request to a volume server to upload the content
func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
return upload_content(uploadUrl, func(w io.Writer) (err error) {
_, err = io.Copy(w, reader)
return
}, filename, isGzipped, mtype, pairMap, jwt)
// Upload sends a POST request to a volume server to upload the content with adjustable compression level
func UploadWithLocalCompressionLevel(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt, compressionLevel int) (*UploadResult, error) {
if compressionLevel < 1 {
compressionLevel = 1
}
if compressionLevel > 9 {
compressionLevel = 9
}
return doUpload(uploadUrl, filename, reader, isGzipped, mtype, pairMap, compressionLevel, jwt)
}
// Upload sends a POST request to a volume server to upload the content with fast compression
func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
return doUpload(uploadUrl, filename, reader, isGzipped, mtype, pairMap, flate.BestSpeed, jwt)
}
func doUpload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, compression int, jwt security.EncodedJwt) (*UploadResult, error) {
contentIsGzipped := isGzipped
shouldGzipNow := false
if !isGzipped {
if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeZipped {
shouldGzipNow = true
contentIsGzipped = true
}
}
return upload_content(uploadUrl, func(w io.Writer) (err error) {
if shouldGzipNow {
gzWriter, _ := gzip.NewWriterLevel(w, compression)
_, err = io.Copy(gzWriter, reader)
gzWriter.Close()
} else {
_, err = io.Copy(w, reader)
}
return
}, filename, contentIsGzipped, mtype, pairMap, jwt)
}
func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
body_buf := bytes.NewBufferString("")
body_writer := multipart.NewWriter(body_buf)
@ -58,9 +91,6 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
if isGzipped {
h.Set("Content-Encoding", "gzip")
}
if jwt != "" {
h.Set("Authorization", "BEARER "+string(jwt))
}
file_writer, cp_err := body_writer.CreatePart(h)
if cp_err != nil {
@ -86,18 +116,15 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
for k, v := range pairMap {
req.Header.Set(k, v)
}
if jwt != "" {
req.Header.Set("Authorization", "BEARER "+string(jwt))
}
resp, post_err := client.Do(req)
if post_err != nil {
glog.V(0).Infoln("failing to upload to", uploadUrl, post_err.Error())
return nil, post_err
}
defer resp.Body.Close()
if resp.StatusCode < http.StatusOK ||
resp.StatusCode > http.StatusIMUsed {
return nil, errors.New(http.StatusText(resp.StatusCode))
}
etag := getEtag(resp)
resp_body, ra_err := ioutil.ReadAll(resp.Body)
if ra_err != nil {

44
weed/pb/filer.proto
View File

@ -24,6 +24,9 @@ service SeaweedFiler {
rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
}
rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
}
rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
}
@ -36,6 +39,9 @@ service SeaweedFiler {
rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
}
rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) {
}
}
//////////////////////////////////////////////////
@ -69,19 +75,33 @@ message Entry {
map<string, bytes> extended = 5;
}
message FullEntry {
string dir = 1;
Entry entry = 2;
}
message EventNotification {
Entry old_entry = 1;
Entry new_entry = 2;
bool delete_chunks = 3;
string new_parent_path = 4;
}
message FileChunk {
string file_id = 1;
string file_id = 1; // to be deprecated
int64 offset = 2;
uint64 size = 3;
int64 mtime = 4;
string e_tag = 5;
string source_file_id = 6;
string source_file_id = 6; // to be deprecated
FileId fid = 7;
FileId source_fid = 8;
}
message FileId {
uint32 volume_id = 1;
uint64 file_key = 2;
fixed32 cookie = 3;
}
message FuseAttributes {
@ -126,6 +146,16 @@ message DeleteEntryRequest {
message DeleteEntryResponse {
}
message AtomicRenameEntryRequest {
string old_directory = 1;
string old_name = 2;
string new_directory = 3;
string new_name = 4;
}
message AtomicRenameEntryResponse {
}
message AssignVolumeRequest {
int32 count = 1;
string collection = 2;
@ -139,6 +169,7 @@ message AssignVolumeResponse {
string url = 2;
string public_url = 3;
int32 count = 4;
string auth = 5;
}
message LookupVolumeRequest {
@ -177,3 +208,12 @@ message StatisticsResponse {
uint64 used_size = 5;
uint64 file_count = 6;
}
message GetFilerConfigurationRequest {
}
message GetFilerConfigurationResponse {
repeated string masters = 1;
string replication = 2;
string collection = 3;
uint32 max_mb = 4;
}

482
weed/pb/filer_pb/filer.pb.go
View File

@ -14,8 +14,10 @@ It has these top-level messages:
ListEntriesRequest
ListEntriesResponse
Entry
FullEntry
EventNotification
FileChunk
FileId
FuseAttributes
CreateEntryRequest
CreateEntryResponse
@ -23,6 +25,8 @@ It has these top-level messages:
UpdateEntryResponse
DeleteEntryRequest
DeleteEntryResponse
AtomicRenameEntryRequest
AtomicRenameEntryResponse
AssignVolumeRequest
AssignVolumeResponse
LookupVolumeRequest
@ -33,6 +37,8 @@ It has these top-level messages:
DeleteCollectionResponse
StatisticsRequest
StatisticsResponse
GetFilerConfigurationRequest
GetFilerConfigurationResponse
*/
package filer_pb
@ -208,16 +214,41 @@ func (m *Entry) GetExtended() map[string][]byte {
return nil
}
type FullEntry struct {
Dir string `protobuf:"bytes,1,opt,name=dir" json:"dir,omitempty"`
Entry *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"`
}
func (m *FullEntry) Reset() { *m = FullEntry{} }
func (m *FullEntry) String() string { return proto.CompactTextString(m) }
func (*FullEntry) ProtoMessage() {}
func (*FullEntry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
func (m *FullEntry) GetDir() string {
if m != nil {
return m.Dir
}
return ""
}
func (m *FullEntry) GetEntry() *Entry {
if m != nil {
return m.Entry
}
return nil
}
type EventNotification struct {
OldEntry *Entry `protobuf:"bytes,1,opt,name=old_entry,json=oldEntry" json:"old_entry,omitempty"`
NewEntry *Entry `protobuf:"bytes,2,opt,name=new_entry,json=newEntry" json:"new_entry,omitempty"`
DeleteChunks bool `protobuf:"varint,3,opt,name=delete_chunks,json=deleteChunks" json:"delete_chunks,omitempty"`
NewParentPath string `protobuf:"bytes,4,opt,name=new_parent_path,json=newParentPath" json:"new_parent_path,omitempty"`
}
func (m *EventNotification) Reset() { *m = EventNotification{} }
func (m *EventNotification) String() string { return proto.CompactTextString(m) }
func (*EventNotification) ProtoMessage() {}
func (*EventNotification) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
func (*EventNotification) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
func (m *EventNotification) GetOldEntry() *Entry {
if m != nil {
@ -240,6 +271,13 @@ func (m *EventNotification) GetDeleteChunks() bool {
return false
}
func (m *EventNotification) GetNewParentPath() string {
if m != nil {
return m.NewParentPath
}
return ""
}
type FileChunk struct {
FileId string `protobuf:"bytes,1,opt,name=file_id,json=fileId" json:"file_id,omitempty"`
Offset int64 `protobuf:"varint,2,opt,name=offset" json:"offset,omitempty"`
@ -247,12 +285,14 @@ type FileChunk struct {
Mtime int64 `protobuf:"varint,4,opt,name=mtime" json:"mtime,omitempty"`
ETag string `protobuf:"bytes,5,opt,name=e_tag,json=eTag" json:"e_tag,omitempty"`
SourceFileId string `protobuf:"bytes,6,opt,name=source_file_id,json=sourceFileId" json:"source_file_id,omitempty"`
Fid *FileId `protobuf:"bytes,7,opt,name=fid" json:"fid,omitempty"`
SourceFid *FileId `protobuf:"bytes,8,opt,name=source_fid,json=sourceFid" json:"source_fid,omitempty"`
}
func (m *FileChunk) Reset() { *m = FileChunk{} }
func (m *FileChunk) String() string { return proto.CompactTextString(m) }
func (*FileChunk) ProtoMessage() {}
func (*FileChunk) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
func (*FileChunk) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
func (m *FileChunk) GetFileId() string {
if m != nil {
@ -296,6 +336,52 @@ func (m *FileChunk) GetSourceFileId() string {
return ""
}
func (m *FileChunk) GetFid() *FileId {
if m != nil {
return m.Fid
}
return nil
}
func (m *FileChunk) GetSourceFid() *FileId {
if m != nil {
return m.SourceFid
}
return nil
}
type FileId struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
FileKey uint64 `protobuf:"varint,2,opt,name=file_key,json=fileKey" json:"file_key,omitempty"`
Cookie uint32 `protobuf:"fixed32,3,opt,name=cookie" json:"cookie,omitempty"`
}
func (m *FileId) Reset() { *m = FileId{} }
func (m *FileId) String() string { return proto.CompactTextString(m) }
func (*FileId) ProtoMessage() {}
func (*FileId) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
func (m *FileId) GetVolumeId() uint32 {
if m != nil {
return m.VolumeId
}
return 0
}
func (m *FileId) GetFileKey() uint64 {
if m != nil {
return m.FileKey
}
return 0
}
func (m *FileId) GetCookie() uint32 {
if m != nil {
return m.Cookie
}
return 0
}
type FuseAttributes struct {
FileSize uint64 `protobuf:"varint,1,opt,name=file_size,json=fileSize" json:"file_size,omitempty"`
Mtime int64 `protobuf:"varint,2,opt,name=mtime" json:"mtime,omitempty"`
@ -315,7 +401,7 @@ type FuseAttributes struct {
func (m *FuseAttributes) Reset() { *m = FuseAttributes{} }
func (m *FuseAttributes) String() string { return proto.CompactTextString(m) }
func (*FuseAttributes) ProtoMessage() {}
func (*FuseAttributes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
func (*FuseAttributes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
func (m *FuseAttributes) GetFileSize() uint64 {
if m != nil {
@ -416,7 +502,7 @@ type CreateEntryRequest struct {
func (m *CreateEntryRequest) Reset() { *m = CreateEntryRequest{} }
func (m *CreateEntryRequest) String() string { return proto.CompactTextString(m) }
func (*CreateEntryRequest) ProtoMessage() {}
func (*CreateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
func (*CreateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
func (m *CreateEntryRequest) GetDirectory() string {
if m != nil {
@ -438,7 +524,7 @@ type CreateEntryResponse struct {
func (m *CreateEntryResponse) Reset() { *m = CreateEntryResponse{} }
func (m *CreateEntryResponse) String() string { return proto.CompactTextString(m) }
func (*CreateEntryResponse) ProtoMessage() {}
func (*CreateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
func (*CreateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
type UpdateEntryRequest struct {
Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
@ -448,7 +534,7 @@ type UpdateEntryRequest struct {
func (m *UpdateEntryRequest) Reset() { *m = UpdateEntryRequest{} }
func (m *UpdateEntryRequest) String() string { return proto.CompactTextString(m) }
func (*UpdateEntryRequest) ProtoMessage() {}
func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
func (m *UpdateEntryRequest) GetDirectory() string {
if m != nil {
@ -470,7 +556,7 @@ type UpdateEntryResponse struct {
func (m *UpdateEntryResponse) Reset() { *m = UpdateEntryResponse{} }
func (m *UpdateEntryResponse) String() string { return proto.CompactTextString(m) }
func (*UpdateEntryResponse) ProtoMessage() {}
func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
type DeleteEntryRequest struct {
Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
@ -483,7 +569,7 @@ type DeleteEntryRequest struct {
func (m *DeleteEntryRequest) Reset() { *m = DeleteEntryRequest{} }
func (m *DeleteEntryRequest) String() string { return proto.CompactTextString(m) }
func (*DeleteEntryRequest) ProtoMessage() {}
func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
func (m *DeleteEntryRequest) GetDirectory() string {
if m != nil {
@ -519,7 +605,55 @@ type DeleteEntryResponse struct {
func (m *DeleteEntryResponse) Reset() { *m = DeleteEntryResponse{} }
func (m *DeleteEntryResponse) String() string { return proto.CompactTextString(m) }
func (*DeleteEntryResponse) ProtoMessage() {}
func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
type AtomicRenameEntryRequest struct {
OldDirectory string `protobuf:"bytes,1,opt,name=old_directory,json=oldDirectory" json:"old_directory,omitempty"`
OldName string `protobuf:"bytes,2,opt,name=old_name,json=oldName" json:"old_name,omitempty"`
NewDirectory string `protobuf:"bytes,3,opt,name=new_directory,json=newDirectory" json:"new_directory,omitempty"`
NewName string `protobuf:"bytes,4,opt,name=new_name,json=newName" json:"new_name,omitempty"`
}
func (m *AtomicRenameEntryRequest) Reset() { *m = AtomicRenameEntryRequest{} }
func (m *AtomicRenameEntryRequest) String() string { return proto.CompactTextString(m) }
func (*AtomicRenameEntryRequest) ProtoMessage() {}
func (*AtomicRenameEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
func (m *AtomicRenameEntryRequest) GetOldDirectory() string {
if m != nil {
return m.OldDirectory
}
return ""
}
func (m *AtomicRenameEntryRequest) GetOldName() string {
if m != nil {
return m.OldName
}
return ""
}
func (m *AtomicRenameEntryRequest) GetNewDirectory() string {
if m != nil {
return m.NewDirectory
}
return ""
}
func (m *AtomicRenameEntryRequest) GetNewName() string {
if m != nil {
return m.NewName
}
return ""
}
type AtomicRenameEntryResponse struct {
}
func (m *AtomicRenameEntryResponse) Reset() { *m = AtomicRenameEntryResponse{} }
func (m *AtomicRenameEntryResponse) String() string { return proto.CompactTextString(m) }
func (*AtomicRenameEntryResponse) ProtoMessage() {}
func (*AtomicRenameEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
type AssignVolumeRequest struct {
Count int32 `protobuf:"varint,1,opt,name=count" json:"count,omitempty"`
@ -532,7 +666,7 @@ type AssignVolumeRequest struct {
func (m *AssignVolumeRequest) Reset() { *m = AssignVolumeRequest{} }
func (m *AssignVolumeRequest) String() string { return proto.CompactTextString(m) }
func (*AssignVolumeRequest) ProtoMessage() {}
func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
func (m *AssignVolumeRequest) GetCount() int32 {
if m != nil {
@ -574,12 +708,13 @@ type AssignVolumeResponse struct {
Url string `protobuf:"bytes,2,opt,name=url" json:"url,omitempty"`
PublicUrl string `protobuf:"bytes,3,opt,name=public_url,json=publicUrl" json:"public_url,omitempty"`
Count int32 `protobuf:"varint,4,opt,name=count" json:"count,omitempty"`
Auth string `protobuf:"bytes,5,opt,name=auth" json:"auth,omitempty"`
}
func (m *AssignVolumeResponse) Reset() { *m = AssignVolumeResponse{} }
func (m *AssignVolumeResponse) String() string { return proto.CompactTextString(m) }
func (*AssignVolumeResponse) ProtoMessage() {}
func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
func (m *AssignVolumeResponse) GetFileId() string {
if m != nil {
@ -609,6 +744,13 @@ func (m *AssignVolumeResponse) GetCount() int32 {
return 0
}
func (m *AssignVolumeResponse) GetAuth() string {
if m != nil {
return m.Auth
}
return ""
}
type LookupVolumeRequest struct {
VolumeIds []string `protobuf:"bytes,1,rep,name=volume_ids,json=volumeIds" json:"volume_ids,omitempty"`
}
@ -616,7 +758,7 @@ type LookupVolumeRequest struct {
func (m *LookupVolumeRequest) Reset() { *m = LookupVolumeRequest{} }
func (m *LookupVolumeRequest) String() string { return proto.CompactTextString(m) }
func (*LookupVolumeRequest) ProtoMessage() {}
func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
func (m *LookupVolumeRequest) GetVolumeIds() []string {
if m != nil {
@ -632,7 +774,7 @@ type Locations struct {
func (m *Locations) Reset() { *m = Locations{} }
func (m *Locations) String() string { return proto.CompactTextString(m) }
func (*Locations) ProtoMessage() {}
func (*Locations) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
func (*Locations) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
func (m *Locations) GetLocations() []*Location {
if m != nil {
@ -649,7 +791,7 @@ type Location struct {
func (m *Location) Reset() { *m = Location{} }
func (m *Location) String() string { return proto.CompactTextString(m) }
func (*Location) ProtoMessage() {}
func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
func (m *Location) GetUrl() string {
if m != nil {
@ -672,7 +814,7 @@ type LookupVolumeResponse struct {
func (m *LookupVolumeResponse) Reset() { *m = LookupVolumeResponse{} }
func (m *LookupVolumeResponse) String() string { return proto.CompactTextString(m) }
func (*LookupVolumeResponse) ProtoMessage() {}
func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
func (m *LookupVolumeResponse) GetLocationsMap() map[string]*Locations {
if m != nil {
@ -688,7 +830,7 @@ type DeleteCollectionRequest struct {
func (m *DeleteCollectionRequest) Reset() { *m = DeleteCollectionRequest{} }
func (m *DeleteCollectionRequest) String() string { return proto.CompactTextString(m) }
func (*DeleteCollectionRequest) ProtoMessage() {}
func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
func (m *DeleteCollectionRequest) GetCollection() string {
if m != nil {
@ -703,7 +845,7 @@ type DeleteCollectionResponse struct {
func (m *DeleteCollectionResponse) Reset() { *m = DeleteCollectionResponse{} }
func (m *DeleteCollectionResponse) String() string { return proto.CompactTextString(m) }
func (*DeleteCollectionResponse) ProtoMessage() {}
func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
type StatisticsRequest struct {
Replication string `protobuf:"bytes,1,opt,name=replication" json:"replication,omitempty"`
@ -714,7 +856,7 @@ type StatisticsRequest struct {
func (m *StatisticsRequest) Reset() { *m = StatisticsRequest{} }
func (m *StatisticsRequest) String() string { return proto.CompactTextString(m) }
func (*StatisticsRequest) ProtoMessage() {}
func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
func (m *StatisticsRequest) GetReplication() string {
if m != nil {
@ -749,7 +891,7 @@ type StatisticsResponse struct {
func (m *StatisticsResponse) Reset() { *m = StatisticsResponse{} }
func (m *StatisticsResponse) String() string { return proto.CompactTextString(m) }
func (*StatisticsResponse) ProtoMessage() {}
func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
func (m *StatisticsResponse) GetReplication() string {
if m != nil {
@ -793,14 +935,64 @@ func (m *StatisticsResponse) GetFileCount() uint64 {
return 0
}
type GetFilerConfigurationRequest struct {
}
func (m *GetFilerConfigurationRequest) Reset() { *m = GetFilerConfigurationRequest{} }
func (m *GetFilerConfigurationRequest) String() string { return proto.CompactTextString(m) }
func (*GetFilerConfigurationRequest) ProtoMessage() {}
func (*GetFilerConfigurationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} }
type GetFilerConfigurationResponse struct {
Masters []string `protobuf:"bytes,1,rep,name=masters" json:"masters,omitempty"`
Replication string `protobuf:"bytes,2,opt,name=replication" json:"replication,omitempty"`
Collection string `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"`
MaxMb uint32 `protobuf:"varint,4,opt,name=max_mb,json=maxMb" json:"max_mb,omitempty"`
}
func (m *GetFilerConfigurationResponse) Reset() { *m = GetFilerConfigurationResponse{} }
func (m *GetFilerConfigurationResponse) String() string { return proto.CompactTextString(m) }
func (*GetFilerConfigurationResponse) ProtoMessage() {}
func (*GetFilerConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} }
func (m *GetFilerConfigurationResponse) GetMasters() []string {
if m != nil {
return m.Masters
}
return nil
}
func (m *GetFilerConfigurationResponse) GetReplication() string {
if m != nil {
return m.Replication
}
return ""
}
func (m *GetFilerConfigurationResponse) GetCollection() string {
if m != nil {
return m.Collection
}
return ""
}
func (m *GetFilerConfigurationResponse) GetMaxMb() uint32 {
if m != nil {
return m.MaxMb
}
return 0
}
func init() {
proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest")
proto.RegisterType((*LookupDirectoryEntryResponse)(nil), "filer_pb.LookupDirectoryEntryResponse")
proto.RegisterType((*ListEntriesRequest)(nil), "filer_pb.ListEntriesRequest")
proto.RegisterType((*ListEntriesResponse)(nil), "filer_pb.ListEntriesResponse")
proto.RegisterType((*Entry)(nil), "filer_pb.Entry")
proto.RegisterType((*FullEntry)(nil), "filer_pb.FullEntry")
proto.RegisterType((*EventNotification)(nil), "filer_pb.EventNotification")
proto.RegisterType((*FileChunk)(nil), "filer_pb.FileChunk")
proto.RegisterType((*FileId)(nil), "filer_pb.FileId")
proto.RegisterType((*FuseAttributes)(nil), "filer_pb.FuseAttributes")
proto.RegisterType((*CreateEntryRequest)(nil), "filer_pb.CreateEntryRequest")
proto.RegisterType((*CreateEntryResponse)(nil), "filer_pb.CreateEntryResponse")
@ -808,6 +1000,8 @@ func init() {
proto.RegisterType((*UpdateEntryResponse)(nil), "filer_pb.UpdateEntryResponse")
proto.RegisterType((*DeleteEntryRequest)(nil), "filer_pb.DeleteEntryRequest")
proto.RegisterType((*DeleteEntryResponse)(nil), "filer_pb.DeleteEntryResponse")
proto.RegisterType((*AtomicRenameEntryRequest)(nil), "filer_pb.AtomicRenameEntryRequest")
proto.RegisterType((*AtomicRenameEntryResponse)(nil), "filer_pb.AtomicRenameEntryResponse")
proto.RegisterType((*AssignVolumeRequest)(nil), "filer_pb.AssignVolumeRequest")
proto.RegisterType((*AssignVolumeResponse)(nil), "filer_pb.AssignVolumeResponse")
proto.RegisterType((*LookupVolumeRequest)(nil), "filer_pb.LookupVolumeRequest")
@ -818,6 +1012,8 @@ func init() {
proto.RegisterType((*DeleteCollectionResponse)(nil), "filer_pb.DeleteCollectionResponse")
proto.RegisterType((*StatisticsRequest)(nil), "filer_pb.StatisticsRequest")
proto.RegisterType((*StatisticsResponse)(nil), "filer_pb.StatisticsResponse")
proto.RegisterType((*GetFilerConfigurationRequest)(nil), "filer_pb.GetFilerConfigurationRequest")
proto.RegisterType((*GetFilerConfigurationResponse)(nil), "filer_pb.GetFilerConfigurationResponse")
}
// Reference imports to suppress errors if they are not otherwise used.
@ -836,10 +1032,12 @@ type SeaweedFilerClient interface {
CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error)
UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error)
DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error)
AtomicRenameEntry(ctx context.Context, in *AtomicRenameEntryRequest, opts ...grpc.CallOption) (*AtomicRenameEntryResponse, error)
AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error)
LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error)
DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error)
Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error)
GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error)
}
type seaweedFilerClient struct {
@ -895,6 +1093,15 @@ func (c *seaweedFilerClient) DeleteEntry(ctx context.Context, in *DeleteEntryReq
return out, nil
}
func (c *seaweedFilerClient) AtomicRenameEntry(ctx context.Context, in *AtomicRenameEntryRequest, opts ...grpc.CallOption) (*AtomicRenameEntryResponse, error) {
out := new(AtomicRenameEntryResponse)
err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/AtomicRenameEntry", in, out, c.cc, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *seaweedFilerClient) AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) {
out := new(AssignVolumeResponse)
err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/AssignVolume", in, out, c.cc, opts...)
@ -931,6 +1138,15 @@ func (c *seaweedFilerClient) Statistics(ctx context.Context, in *StatisticsReque
return out, nil
}
func (c *seaweedFilerClient) GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error) {
out := new(GetFilerConfigurationResponse)
err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/GetFilerConfiguration", in, out, c.cc, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// Server API for SeaweedFiler service
type SeaweedFilerServer interface {
@ -939,10 +1155,12 @@ type SeaweedFilerServer interface {
CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error)
UpdateEntry(context.Context, *UpdateEntryRequest) (*UpdateEntryResponse, error)
DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error)
AtomicRenameEntry(context.Context, *AtomicRenameEntryRequest) (*AtomicRenameEntryResponse, error)
AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error)
LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error)
DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error)
Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error)
GetFilerConfiguration(context.Context, *GetFilerConfigurationRequest) (*GetFilerConfigurationResponse, error)
}
func RegisterSeaweedFilerServer(s *grpc.Server, srv SeaweedFilerServer) {
@ -1039,6 +1257,24 @@ func _SeaweedFiler_DeleteEntry_Handler(srv interface{}, ctx context.Context, dec
return interceptor(ctx, in, info, handler)
}
func _SeaweedFiler_AtomicRenameEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(AtomicRenameEntryRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SeaweedFilerServer).AtomicRenameEntry(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/filer_pb.SeaweedFiler/AtomicRenameEntry",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedFilerServer).AtomicRenameEntry(ctx, req.(*AtomicRenameEntryRequest))
}
return interceptor(ctx, in, info, handler)
}
func _SeaweedFiler_AssignVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(AssignVolumeRequest)
if err := dec(in); err != nil {
@ -1111,6 +1347,24 @@ func _SeaweedFiler_Statistics_Handler(srv interface{}, ctx context.Context, dec
return interceptor(ctx, in, info, handler)
}
func _SeaweedFiler_GetFilerConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(GetFilerConfigurationRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SeaweedFilerServer).GetFilerConfiguration(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/filer_pb.SeaweedFiler/GetFilerConfiguration",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedFilerServer).GetFilerConfiguration(ctx, req.(*GetFilerConfigurationRequest))
}
return interceptor(ctx, in, info, handler)
}
var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
ServiceName: "filer_pb.SeaweedFiler",
HandlerType: (*SeaweedFilerServer)(nil),
@ -1135,6 +1389,10 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
MethodName: "DeleteEntry",
Handler: _SeaweedFiler_DeleteEntry_Handler,
},
{
MethodName: "AtomicRenameEntry",
Handler: _SeaweedFiler_AtomicRenameEntry_Handler,
},
{
MethodName: "AssignVolume",
Handler: _SeaweedFiler_AssignVolume_Handler,
@ -1151,6 +1409,10 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
MethodName: "Statistics",
Handler: _SeaweedFiler_Statistics_Handler,
},
{
MethodName: "GetFilerConfiguration",
Handler: _SeaweedFiler_GetFilerConfiguration_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "filer.proto",
@ -1159,86 +1421,104 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 1291 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x57, 0x4d, 0x8f, 0xdc, 0x44,
0x13, 0x8e, 0xe7, 0x2b, 0xe3, 0x9a, 0x99, 0xbc, 0xbb, 0x3d, 0xfb, 0x12, 0x6b, 0xb2, 0x1b, 0x26,
0x86, 0xa0, 0x8d, 0x88, 0x46, 0x51, 0xe0, 0x90, 0x10, 0x21, 0x91, 0x6c, 0x36, 0x52, 0xa4, 0x4d,
0x82, 0xbc, 0x09, 0x12, 0xe2, 0x60, 0x79, 0xed, 0x9e, 0xa1, 0xb5, 0x1e, 0x7b, 0x70, 0xb7, 0x37,
0x09, 0x7f, 0x82, 0x0b, 0x57, 0x0e, 0x9c, 0xf8, 0x17, 0x5c, 0xf8, 0x3f, 0xdc, 0xb9, 0xa1, 0xae,
0x6e, 0x7b, 0xda, 0x63, 0xef, 0x06, 0x84, 0x72, 0xeb, 0x7e, 0xaa, 0xba, 0xbe, 0xfa, 0xe9, 0x2a,
0x1b, 0x06, 0x73, 0x16, 0xd3, 0x6c, 0xb6, 0xca, 0x52, 0x91, 0x92, 0x3e, 0x6e, 0xfc, 0xd5, 0x89,
0xfb, 0x02, 0xae, 0x1d, 0xa5, 0xe9, 0x69, 0xbe, 0x7a, 0xcc, 0x32, 0x1a, 0x8a, 0x34, 0x7b, 0x7b,
0x98, 0x88, 0xec, 0xad, 0x47, 0x7f, 0xc8, 0x29, 0x17, 0x64, 0x17, 0xec, 0xa8, 0x10, 0x38, 0xd6,
0xd4, 0xda, 0xb7, 0xbd, 0x35, 0x40, 0x08, 0x74, 0x92, 0x60, 0x49, 0x9d, 0x16, 0x0a, 0x70, 0xed,
0x1e, 0xc2, 0x6e, 0xb3, 0x41, 0xbe, 0x4a, 0x13, 0x4e, 0xc9, 0x4d, 0xe8, 0x52, 0x09, 0xa0, 0xb5,
0xc1, 0xdd, 0xff, 0xcd, 0x8a, 0x50, 0x66, 0x4a, 0x4f, 0x49, 0xdd, 0xdf, 0x2d, 0x20, 0x47, 0x8c,
0x0b, 0x09, 0x32, 0xca, 0xff, 0x59, 0x3c, 0x1f, 0x40, 0x6f, 0x95, 0xd1, 0x39, 0x7b, 0xa3, 0x23,
0xd2, 0x3b, 0x72, 0x1b, 0xb6, 0xb9, 0x08, 0x32, 0xf1, 0x24, 0x4b, 0x97, 0x4f, 0x58, 0x4c, 0x9f,
0xcb, 0xa0, 0xdb, 0xa8, 0x52, 0x17, 0x90, 0x19, 0x10, 0x96, 0x84, 0x71, 0xce, 0xd9, 0x19, 0x3d,
0x2e, 0xa4, 0x4e, 0x67, 0x6a, 0xed, 0xf7, 0xbd, 0x06, 0x09, 0xd9, 0x81, 0x6e, 0xcc, 0x96, 0x4c,
0x38, 0xdd, 0xa9, 0xb5, 0x3f, 0xf2, 0xd4, 0xc6, 0xfd, 0x0a, 0xc6, 0x95, 0xf8, 0x75, 0xfa, 0xb7,
0xe0, 0x32, 0x55, 0x90, 0x63, 0x4d, 0xdb, 0x4d, 0x05, 0x28, 0xe4, 0xee, 0x2f, 0x2d, 0xe8, 0x22,
0x54, 0xd6, 0xd9, 0x5a, 0xd7, 0x99, 0xdc, 0x80, 0x21, 0xe3, 0xfe, 0xba, 0x18, 0x2d, 0x8c, 0x6f,
0xc0, 0x78, 0x59, 0x77, 0xf2, 0x29, 0xf4, 0xc2, 0xef, 0xf3, 0xe4, 0x94, 0x3b, 0x6d, 0x74, 0x35,
0x5e, 0xbb, 0x92, 0xc9, 0x1e, 0x48, 0x99, 0xa7, 0x55, 0xc8, 0x3d, 0x80, 0x40, 0x88, 0x8c, 0x9d,
0xe4, 0x82, 0x72, 0xcc, 0x76, 0x70, 0xd7, 0x31, 0x0e, 0xe4, 0x9c, 0x3e, 0x2c, 0xe5, 0x9e, 0xa1,
0x4b, 0xee, 0x43, 0x9f, 0xbe, 0x11, 0x34, 0x89, 0x68, 0xe4, 0x74, 0xd1, 0xd1, 0xde, 0x46, 0x4e,
0xb3, 0x43, 0x2d, 0x57, 0x19, 0x96, 0xea, 0x93, 0x07, 0x30, 0xaa, 0x88, 0xc8, 0x16, 0xb4, 0x4f,
0x69, 0x71, 0xb3, 0x72, 0x29, 0xab, 0x7b, 0x16, 0xc4, 0xb9, 0x22, 0xd9, 0xd0, 0x53, 0x9b, 0x2f,
0x5a, 0xf7, 0x2c, 0xf7, 0x67, 0x0b, 0xb6, 0x0f, 0xcf, 0x68, 0x22, 0x9e, 0xa7, 0x82, 0xcd, 0x59,
0x18, 0x08, 0x96, 0x26, 0xe4, 0x36, 0xd8, 0x69, 0x1c, 0xf9, 0x17, 0x72, 0xac, 0x9f, 0xc6, 0xda,
0xdf, 0x6d, 0xb0, 0x13, 0xfa, 0x5a, 0x6b, 0xb7, 0xce, 0xd1, 0x4e, 0xe8, 0x6b, 0xa5, 0xfd, 0x11,
0x8c, 0x22, 0x1a, 0x53, 0x41, 0xfd, 0xb2, 0xae, 0xb2, 0xe8, 0x43, 0x05, 0x62, 0x3d, 0xb9, 0xfb,
0xab, 0x05, 0x76, 0x59, 0x5e, 0x72, 0x15, 0x2e, 0x4b, 0x73, 0x3e, 0x8b, 0x74, 0x52, 0x3d, 0xb9,
0x7d, 0x1a, 0x49, 0xae, 0xa6, 0xf3, 0x39, 0xa7, 0x02, 0xdd, 0xb6, 0x3d, 0xbd, 0x93, 0x77, 0xcd,
0xd9, 0x8f, 0x8a, 0x9e, 0x1d, 0x0f, 0xd7, 0xb2, 0x06, 0x4b, 0xc1, 0x96, 0x14, 0xaf, 0xa5, 0xed,
0xa9, 0x0d, 0x19, 0x43, 0x97, 0xfa, 0x22, 0x58, 0x20, 0xef, 0x6c, 0xaf, 0x43, 0x5f, 0x06, 0x0b,
0xf2, 0x31, 0x5c, 0xe1, 0x69, 0x9e, 0x85, 0xd4, 0x2f, 0xdc, 0xf6, 0x50, 0x3a, 0x54, 0xe8, 0x13,
0x74, 0xee, 0xfe, 0xd9, 0x82, 0x2b, 0xd5, 0x1b, 0x25, 0xd7, 0xc0, 0xc6, 0x13, 0xe8, 0xdc, 0x42,
0xe7, 0xd8, 0x25, 0x8e, 0x2b, 0x01, 0xb4, 0xcc, 0x00, 0x8a, 0x23, 0xcb, 0x34, 0x52, 0xf1, 0x8e,
0xd4, 0x91, 0x67, 0x69, 0x44, 0xe5, 0x4d, 0xe6, 0x2c, 0xc2, 0x88, 0x47, 0x9e, 0x5c, 0x4a, 0x64,
0xc1, 0x22, 0xfd, 0x4a, 0xe4, 0x52, 0xd6, 0x20, 0xcc, 0xd0, 0x6e, 0x4f, 0xd5, 0x40, 0xed, 0x64,
0x0d, 0x96, 0x12, 0xbd, 0xac, 0x12, 0x93, 0x6b, 0x32, 0x85, 0x41, 0x46, 0x57, 0xb1, 0xbe, 0x66,
0xa7, 0x8f, 0x22, 0x13, 0x22, 0xd7, 0x01, 0xc2, 0x34, 0x8e, 0x69, 0x88, 0x0a, 0x36, 0x2a, 0x18,
0x88, 0xbc, 0x0a, 0x21, 0x62, 0x9f, 0xd3, 0xd0, 0x81, 0xa9, 0xb5, 0xdf, 0xf5, 0x7a, 0x42, 0xc4,
0xc7, 0x34, 0x94, 0x79, 0xe4, 0x9c, 0x66, 0x3e, 0xbe, 0xb1, 0x01, 0x9e, 0xeb, 0x4b, 0x00, 0xbb,
0xc1, 0x1e, 0xc0, 0x22, 0x4b, 0xf3, 0x95, 0x92, 0x0e, 0xa7, 0x6d, 0xd9, 0x72, 0x10, 0x41, 0xf1,
0x4d, 0xb8, 0xc2, 0xdf, 0x2e, 0x63, 0x96, 0x9c, 0xfa, 0x22, 0xc8, 0x16, 0x54, 0x38, 0x23, 0x34,
0x30, 0xd2, 0xe8, 0x4b, 0x04, 0xdd, 0x6f, 0x81, 0x1c, 0x64, 0x34, 0x10, 0xf4, 0x5f, 0x74, 0xd7,
0xb2, 0x53, 0xb6, 0x2e, 0xec, 0x94, 0xff, 0x87, 0x71, 0xc5, 0xb4, 0x6a, 0x34, 0xd2, 0xe3, 0xab,
0x55, 0xf4, 0xbe, 0x3c, 0x56, 0x4c, 0x6b, 0x8f, 0x3f, 0x59, 0x40, 0x1e, 0xe3, 0x4b, 0xf8, 0x6f,
0x23, 0x44, 0x72, 0x58, 0xb6, 0x36, 0xf5, 0xd2, 0xa2, 0x40, 0x04, 0xba, 0xf9, 0x0e, 0x19, 0x57,
0xf6, 0x1f, 0x07, 0x22, 0xd0, 0x0d, 0x30, 0xa3, 0x61, 0x9e, 0xc9, 0x7e, 0x8c, 0xbc, 0xc2, 0x06,
0xe8, 0x15, 0x90, 0x0c, 0xb4, 0x12, 0x90, 0x0e, 0xf4, 0x37, 0x0b, 0xc6, 0x0f, 0x39, 0x67, 0x8b,
0xe4, 0x9b, 0x34, 0xce, 0x97, 0xb4, 0x88, 0x74, 0x07, 0xba, 0x61, 0x9a, 0x27, 0x02, 0xa3, 0xec,
0x7a, 0x6a, 0xb3, 0x41, 0xab, 0x56, 0x8d, 0x56, 0x1b, 0xc4, 0x6c, 0xd7, 0x89, 0x69, 0x10, 0xaf,
0x53, 0x21, 0xde, 0x87, 0x30, 0x90, 0xe9, 0xf9, 0x21, 0x4d, 0x04, 0xcd, 0xf4, 0x3b, 0x06, 0x09,
0x1d, 0x20, 0xe2, 0x9e, 0xc1, 0x4e, 0x35, 0x50, 0x3d, 0x45, 0xce, 0xed, 0x2a, 0xf2, 0xd5, 0x65,
0xb1, 0x8e, 0x52, 0x2e, 0x25, 0x7f, 0x57, 0xf9, 0x49, 0xcc, 0x42, 0x5f, 0x0a, 0x54, 0x74, 0xb6,
0x42, 0x5e, 0x65, 0xf1, 0x3a, 0xe7, 0x8e, 0x91, 0xb3, 0xfb, 0x39, 0x8c, 0xd5, 0x10, 0xaf, 0x16,
0x68, 0x0f, 0xe0, 0x0c, 0x01, 0x9f, 0x45, 0x6a, 0x7e, 0xd9, 0x9e, 0xad, 0x90, 0xa7, 0x11, 0x77,
0xbf, 0x04, 0xfb, 0x28, 0x55, 0x39, 0x73, 0x72, 0x07, 0xec, 0xb8, 0xd8, 0xe8, 0x51, 0x47, 0xd6,
0x7c, 0x2a, 0xf4, 0xbc, 0xb5, 0x92, 0xfb, 0x00, 0xfa, 0x05, 0x5c, 0xe4, 0x61, 0x9d, 0x97, 0x47,
0x6b, 0x23, 0x0f, 0xf7, 0x0f, 0x0b, 0x76, 0xaa, 0x21, 0xeb, 0x52, 0xbd, 0x82, 0x51, 0xe9, 0xc2,
0x5f, 0x06, 0x2b, 0x1d, 0xcb, 0x1d, 0x33, 0x96, 0xfa, 0xb1, 0x32, 0x40, 0xfe, 0x2c, 0x58, 0x29,
0xf6, 0x0c, 0x63, 0x03, 0x9a, 0xbc, 0x84, 0xed, 0x9a, 0x4a, 0xc3, 0xf4, 0xba, 0x65, 0x4e, 0xaf,
0xca, 0x04, 0x2e, 0x4f, 0x9b, 0x23, 0xed, 0x3e, 0x5c, 0x55, 0x84, 0x3d, 0x28, 0xf9, 0x55, 0xd4,
0xbe, 0x4a, 0x43, 0x6b, 0x93, 0x86, 0xee, 0x04, 0x9c, 0xfa, 0x51, 0x4d, 0xf8, 0x05, 0x6c, 0x1f,
0x8b, 0x40, 0x30, 0x2e, 0x58, 0x58, 0x7e, 0x4a, 0x6d, 0xf0, 0xd6, 0x7a, 0x57, 0x43, 0xad, 0x33,
0x7f, 0x0b, 0xda, 0x42, 0x14, 0x9c, 0x92, 0x4b, 0x79, 0x0b, 0xc4, 0xf4, 0xa4, 0xef, 0xe0, 0x3d,
0xb8, 0x92, 0x7c, 0x10, 0xa9, 0x08, 0x62, 0x35, 0xb0, 0x3a, 0x38, 0xb0, 0x6c, 0x44, 0x70, 0x62,
0xa9, 0x9e, 0x1e, 0x29, 0x69, 0x57, 0x8d, 0x33, 0x09, 0xa0, 0x70, 0x0f, 0x00, 0x9f, 0x8f, 0x62,
0x7e, 0x4f, 0x9d, 0x95, 0xc8, 0x81, 0x04, 0xee, 0xfe, 0xd5, 0x85, 0xe1, 0x31, 0x0d, 0x5e, 0x53,
0x1a, 0xc9, 0x79, 0x99, 0x91, 0x45, 0xc1, 0xad, 0xea, 0x37, 0x2d, 0xb9, 0xb9, 0x49, 0xa2, 0xc6,
0x8f, 0xe8, 0xc9, 0x27, 0xef, 0x52, 0xd3, 0xd7, 0x74, 0x89, 0x1c, 0xc1, 0xc0, 0xf8, 0x68, 0x24,
0xbb, 0xc6, 0xc1, 0xda, 0xb7, 0xf0, 0x64, 0xef, 0x1c, 0xa9, 0x69, 0xcd, 0x98, 0x0c, 0xa6, 0xb5,
0xfa, 0x2c, 0x32, 0xad, 0x35, 0x8d, 0x13, 0xb4, 0x66, 0x74, 0x7d, 0xd3, 0x5a, 0x7d, 0xce, 0x98,
0xd6, 0x9a, 0x46, 0x05, 0x5a, 0x33, 0x5a, 0xb3, 0x69, 0xad, 0x3e, 0x42, 0x4c, 0x6b, 0x4d, 0xfd,
0xfc, 0x12, 0x79, 0x01, 0x43, 0xb3, 0x4f, 0x12, 0xe3, 0x40, 0x43, 0xa3, 0x9f, 0x5c, 0x3f, 0x4f,
0x6c, 0x1a, 0x34, 0xdb, 0x82, 0x69, 0xb0, 0xa1, 0x31, 0x9a, 0x06, 0x9b, 0xba, 0x89, 0x7b, 0x89,
0x7c, 0x07, 0x5b, 0x9b, 0xcf, 0x93, 0xdc, 0xd8, 0x4c, 0xab, 0xf6, 0xea, 0x27, 0xee, 0x45, 0x2a,
0xa5, 0xf1, 0xa7, 0x00, 0xeb, 0x57, 0x47, 0xae, 0xad, 0xcf, 0xd4, 0x5e, 0xfd, 0x64, 0xb7, 0x59,
0x58, 0x98, 0x7a, 0x74, 0x1d, 0xb6, 0xb8, 0xa2, 0xfe, 0x9c, 0xcf, 0xc2, 0x98, 0xd1, 0x44, 0x3c,
0x02, 0x7c, 0x05, 0x5f, 0xcb, 0x3f, 0xc7, 0x93, 0x1e, 0xfe, 0x40, 0x7e, 0xf6, 0x77, 0x00, 0x00,
0x00, 0xff, 0xff, 0x8d, 0x38, 0xa9, 0x9f, 0x4f, 0x0e, 0x00, 0x00,
// 1583 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xdb, 0x6f, 0xdc, 0x44,
0x17, 0xaf, 0xf7, 0xee, 0xb3, 0xbb, 0x6d, 0x32, 0x49, 0xbf, 0xba, 0x9b, 0xcb, 0x97, 0x3a, 0x5f,
0xfb, 0xa5, 0xa2, 0x0a, 0x55, 0xe1, 0xa1, 0xa5, 0x42, 0xa2, 0xcd, 0x05, 0x45, 0xa4, 0x17, 0x39,
0x2d, 0x02, 0x21, 0x61, 0x39, 0xf6, 0xec, 0x66, 0x88, 0xed, 0x59, 0xec, 0x71, 0x92, 0xf2, 0x27,
0xf0, 0x82, 0xc4, 0x23, 0x12, 0xcf, 0xfc, 0x13, 0x88, 0x17, 0xc4, 0xbf, 0xc3, 0x23, 0xcf, 0x68,
0x2e, 0xf6, 0x8e, 0xd7, 0x9b, 0xa4, 0x08, 0xf5, 0xcd, 0x73, 0xae, 0xbf, 0x73, 0xe6, 0x5c, 0x66,
0x17, 0xba, 0x43, 0x12, 0xe2, 0x64, 0x73, 0x9c, 0x50, 0x46, 0x51, 0x47, 0x1c, 0xdc, 0xf1, 0xa1,
0xfd, 0x02, 0x96, 0xf6, 0x29, 0x3d, 0xce, 0xc6, 0xdb, 0x24, 0xc1, 0x3e, 0xa3, 0xc9, 0x9b, 0x9d,
0x98, 0x25, 0x6f, 0x1c, 0xfc, 0x6d, 0x86, 0x53, 0x86, 0x96, 0xc1, 0x0c, 0x72, 0x86, 0x65, 0xac,
0x19, 0x1b, 0xa6, 0x33, 0x21, 0x20, 0x04, 0x8d, 0xd8, 0x8b, 0xb0, 0x55, 0x13, 0x0c, 0xf1, 0x6d,
0xef, 0xc0, 0xf2, 0x6c, 0x83, 0xe9, 0x98, 0xc6, 0x29, 0x46, 0xb7, 0xa1, 0x89, 0x39, 0x41, 0x58,
0xeb, 0x3e, 0xb8, 0xb6, 0x99, 0x43, 0xd9, 0x94, 0x72, 0x92, 0x6b, 0xff, 0x66, 0x00, 0xda, 0x27,
0x29, 0xe3, 0x44, 0x82, 0xd3, 0xb7, 0xc3, 0xf3, 0x1f, 0x68, 0x8d, 0x13, 0x3c, 0x24, 0x67, 0x0a,
0x91, 0x3a, 0xa1, 0x7b, 0x30, 0x9f, 0x32, 0x2f, 0x61, 0xbb, 0x09, 0x8d, 0x76, 0x49, 0x88, 0x9f,
0x73, 0xd0, 0x75, 0x21, 0x52, 0x65, 0xa0, 0x4d, 0x40, 0x24, 0xf6, 0xc3, 0x2c, 0x25, 0x27, 0xf8,
0x20, 0xe7, 0x5a, 0x8d, 0x35, 0x63, 0xa3, 0xe3, 0xcc, 0xe0, 0xa0, 0x45, 0x68, 0x86, 0x24, 0x22,
0xcc, 0x6a, 0xae, 0x19, 0x1b, 0x7d, 0x47, 0x1e, 0xec, 0x4f, 0x60, 0xa1, 0x84, 0x5f, 0x85, 0x7f,
0x17, 0xda, 0x58, 0x92, 0x2c, 0x63, 0xad, 0x3e, 0x2b, 0x01, 0x39, 0xdf, 0xfe, 0xb9, 0x06, 0x4d,
0x41, 0x2a, 0xf2, 0x6c, 0x4c, 0xf2, 0x8c, 0x6e, 0x41, 0x8f, 0xa4, 0xee, 0x24, 0x19, 0x35, 0x81,
0xaf, 0x4b, 0xd2, 0x22, 0xef, 0xe8, 0x3d, 0x68, 0xf9, 0x47, 0x59, 0x7c, 0x9c, 0x5a, 0x75, 0xe1,
0x6a, 0x61, 0xe2, 0x8a, 0x07, 0xbb, 0xc5, 0x79, 0x8e, 0x12, 0x41, 0x0f, 0x01, 0x3c, 0xc6, 0x12,
0x72, 0x98, 0x31, 0x9c, 0x8a, 0x68, 0xbb, 0x0f, 0x2c, 0x4d, 0x21, 0x4b, 0xf1, 0x93, 0x82, 0xef,
0x68, 0xb2, 0xe8, 0x11, 0x74, 0xf0, 0x19, 0xc3, 0x71, 0x80, 0x03, 0xab, 0x29, 0x1c, 0xad, 0x4c,
0xc5, 0xb4, 0xb9, 0xa3, 0xf8, 0x32, 0xc2, 0x42, 0x7c, 0xf0, 0x18, 0xfa, 0x25, 0x16, 0x9a, 0x83,
0xfa, 0x31, 0xce, 0x6f, 0x96, 0x7f, 0xf2, 0xec, 0x9e, 0x78, 0x61, 0x26, 0x8b, 0xac, 0xe7, 0xc8,
0xc3, 0x47, 0xb5, 0x87, 0x86, 0xbd, 0x0d, 0xe6, 0x6e, 0x16, 0x86, 0x85, 0x62, 0x40, 0x92, 0x5c,
0x31, 0x20, 0xc9, 0xa4, 0xd0, 0x6a, 0x17, 0x16, 0xda, 0xaf, 0x06, 0xcc, 0xef, 0x9c, 0xe0, 0x98,
0x3d, 0xa7, 0x8c, 0x0c, 0x89, 0xef, 0x31, 0x42, 0x63, 0x74, 0x0f, 0x4c, 0x1a, 0x06, 0xee, 0x85,
0x95, 0xda, 0xa1, 0xa1, 0x42, 0x7d, 0x0f, 0xcc, 0x18, 0x9f, 0xba, 0x17, 0xba, 0xeb, 0xc4, 0xf8,
0x54, 0x4a, 0xaf, 0x43, 0x3f, 0xc0, 0x21, 0x66, 0xd8, 0x2d, 0x6e, 0x87, 0x5f, 0x5d, 0x4f, 0x12,
0xb7, 0xe4, 0x75, 0xdc, 0x81, 0x6b, 0xdc, 0xe4, 0xd8, 0x4b, 0x70, 0xcc, 0xdc, 0xb1, 0xc7, 0x8e,
0xc4, 0x9d, 0x98, 0x4e, 0x3f, 0xc6, 0xa7, 0x2f, 0x05, 0xf5, 0xa5, 0xc7, 0x8e, 0xec, 0xbf, 0x0c,
0x30, 0x8b, 0xcb, 0x44, 0x37, 0xa0, 0xcd, 0xdd, 0xba, 0x24, 0x50, 0x99, 0x68, 0xf1, 0xe3, 0x5e,
0xc0, 0x3b, 0x83, 0x0e, 0x87, 0x29, 0x66, 0x02, 0x5e, 0xdd, 0x51, 0x27, 0x5e, 0x59, 0x29, 0xf9,
0x4e, 0x36, 0x43, 0xc3, 0x11, 0xdf, 0x3c, 0xe3, 0x11, 0x23, 0x11, 0x16, 0x0e, 0xeb, 0x8e, 0x3c,
0xa0, 0x05, 0x68, 0x62, 0x97, 0x79, 0x23, 0x51, 0xe5, 0xa6, 0xd3, 0xc0, 0xaf, 0xbc, 0x11, 0xfa,
0x1f, 0x5c, 0x4d, 0x69, 0x96, 0xf8, 0xd8, 0xcd, 0xdd, 0xb6, 0x04, 0xb7, 0x27, 0xa9, 0xbb, 0xd2,
0xb9, 0x0d, 0xf5, 0x21, 0x09, 0xac, 0xb6, 0x48, 0xcc, 0x5c, 0xb9, 0x08, 0xf7, 0x02, 0x87, 0x33,
0xd1, 0xfb, 0x00, 0x85, 0xa5, 0xc0, 0xea, 0x9c, 0x23, 0x6a, 0xe6, 0x76, 0x03, 0xfb, 0x0b, 0x68,
0x29, 0xf3, 0x4b, 0x60, 0x9e, 0xd0, 0x30, 0x8b, 0x8a, 0xb0, 0xfb, 0x4e, 0x47, 0x12, 0xf6, 0x02,
0x74, 0x13, 0xc4, 0xac, 0x73, 0x79, 0x55, 0xd5, 0x44, 0x90, 0x22, 0x43, 0x9f, 0x61, 0x31, 0x2d,
0x7c, 0x4a, 0x8f, 0x89, 0x8c, 0xbe, 0xed, 0xa8, 0x93, 0xfd, 0x67, 0x0d, 0xae, 0x96, 0xcb, 0x9d,
0xbb, 0x10, 0x56, 0x44, 0xae, 0x0c, 0x61, 0x46, 0x98, 0x3d, 0x28, 0xe5, 0xab, 0xa6, 0xe7, 0x2b,
0x57, 0x89, 0x68, 0x20, 0x1d, 0xf4, 0xa5, 0xca, 0x33, 0x1a, 0x60, 0x5e, 0xad, 0x19, 0x09, 0x44,
0x82, 0xfb, 0x0e, 0xff, 0xe4, 0x94, 0x11, 0x09, 0xd4, 0x08, 0xe1, 0x9f, 0x02, 0x5e, 0x22, 0xec,
0xb6, 0xe4, 0x95, 0xc9, 0x13, 0xbf, 0xb2, 0x88, 0x53, 0xdb, 0xf2, 0x1e, 0xf8, 0x37, 0x5a, 0x83,
0x6e, 0x82, 0xc7, 0xa1, 0xaa, 0x5e, 0x91, 0x3e, 0xd3, 0xd1, 0x49, 0x68, 0x15, 0xc0, 0xa7, 0x61,
0x88, 0x7d, 0x21, 0x60, 0x0a, 0x01, 0x8d, 0xc2, 0x2b, 0x87, 0xb1, 0xd0, 0x4d, 0xb1, 0x6f, 0xc1,
0x9a, 0xb1, 0xd1, 0x74, 0x5a, 0x8c, 0x85, 0x07, 0xd8, 0xe7, 0x71, 0x64, 0x29, 0x4e, 0x5c, 0x31,
0x80, 0xba, 0x42, 0xaf, 0xc3, 0x09, 0x62, 0x54, 0xae, 0x00, 0x8c, 0x12, 0x9a, 0x8d, 0x25, 0xb7,
0xb7, 0x56, 0xe7, 0xf3, 0x58, 0x50, 0x04, 0xfb, 0x36, 0x5c, 0x4d, 0xdf, 0x44, 0x21, 0x89, 0x8f,
0x5d, 0xe6, 0x25, 0x23, 0xcc, 0xac, 0xbe, 0xac, 0x61, 0x45, 0x7d, 0x25, 0x88, 0xf6, 0x97, 0x80,
0xb6, 0x12, 0xec, 0x31, 0xfc, 0x0f, 0x56, 0xcf, 0x5b, 0x76, 0xf7, 0x75, 0x58, 0x28, 0x99, 0x96,
0x53, 0x98, 0x7b, 0x7c, 0x3d, 0x0e, 0xde, 0x95, 0xc7, 0x92, 0x69, 0xe5, 0xf1, 0x07, 0x03, 0xd0,
0xb6, 0x68, 0xf0, 0x7f, 0xb7, 0x5f, 0x79, 0xcb, 0xf1, 0xb9, 0x2f, 0x07, 0x48, 0xe0, 0x31, 0x4f,
0x6d, 0xa6, 0x1e, 0x49, 0xa5, 0xfd, 0x6d, 0x8f, 0x79, 0x6a, 0x3b, 0x24, 0xd8, 0xcf, 0x12, 0xbe,
0xac, 0x44, 0x5d, 0x89, 0xed, 0xe0, 0xe4, 0x24, 0x0e, 0xb4, 0x04, 0x48, 0x01, 0xfd, 0xc9, 0x00,
0xeb, 0x09, 0xa3, 0x11, 0xf1, 0x1d, 0xcc, 0x1d, 0x96, 0xe0, 0xae, 0x43, 0x9f, 0x8f, 0xc5, 0x69,
0xc8, 0x3d, 0x1a, 0x06, 0x93, 0xb5, 0x73, 0x13, 0xf8, 0x64, 0x74, 0x35, 0xe4, 0x6d, 0x1a, 0x06,
0xa2, 0x20, 0xd6, 0x81, 0x8f, 0x2f, 0x4d, 0x5f, 0x2e, 0xe1, 0x5e, 0x8c, 0x4f, 0x4b, 0xfa, 0x5c,
0x48, 0xe8, 0xcb, 0x99, 0xd7, 0x8e, 0xf1, 0x29, 0xd7, 0xb7, 0x97, 0xe0, 0xe6, 0x0c, 0x6c, 0x0a,
0xf9, 0x2f, 0x06, 0x2c, 0x3c, 0x49, 0x53, 0x32, 0x8a, 0x3f, 0x17, 0xdd, 0x9f, 0x83, 0x5e, 0x84,
0xa6, 0x4f, 0xb3, 0x98, 0x09, 0xb0, 0x4d, 0x47, 0x1e, 0xa6, 0x1a, 0xa2, 0x56, 0x69, 0x88, 0xa9,
0x96, 0xaa, 0x57, 0x5b, 0x4a, 0x6b, 0x99, 0x46, 0xa9, 0x65, 0xfe, 0x0b, 0x5d, 0x7e, 0x31, 0xae,
0x8f, 0x63, 0x86, 0x13, 0x35, 0x30, 0x81, 0x93, 0xb6, 0x04, 0xc5, 0xfe, 0xde, 0x80, 0xc5, 0x32,
0x52, 0xf5, 0x3a, 0x38, 0x77, 0x7e, 0xf3, 0x81, 0x91, 0x84, 0x0a, 0x26, 0xff, 0xe4, 0xad, 0x37,
0xce, 0x0e, 0x43, 0xe2, 0xbb, 0x9c, 0x21, 0xe1, 0x99, 0x92, 0xf2, 0x3a, 0x09, 0x27, 0x41, 0x37,
0xf4, 0xa0, 0x11, 0x34, 0xbc, 0x8c, 0x1d, 0xe5, 0x33, 0x9c, 0x7f, 0xdb, 0x1f, 0xc2, 0x82, 0x7c,
0xb0, 0x95, 0xb3, 0xb6, 0x02, 0x50, 0x4c, 0x55, 0xf9, 0x56, 0x31, 0x1d, 0x33, 0x1f, 0xab, 0xa9,
0xfd, 0x31, 0x98, 0xfb, 0x54, 0x26, 0x22, 0x45, 0xf7, 0xc1, 0x0c, 0xf3, 0x83, 0x7a, 0xd6, 0xa0,
0x49, 0x7b, 0xe4, 0x72, 0xce, 0x44, 0xc8, 0x7e, 0x0c, 0x9d, 0x9c, 0x9c, 0xc7, 0x66, 0x9c, 0x17,
0x5b, 0x6d, 0x2a, 0x36, 0xfb, 0x77, 0x03, 0x16, 0xcb, 0x90, 0x55, 0xfa, 0x5e, 0x43, 0xbf, 0x70,
0xe1, 0x46, 0xde, 0x58, 0x61, 0xb9, 0xaf, 0x63, 0xa9, 0xaa, 0x15, 0x00, 0xd3, 0x67, 0xde, 0x58,
0x96, 0x54, 0x2f, 0xd4, 0x48, 0x83, 0x57, 0x30, 0x5f, 0x11, 0x99, 0xf1, 0x52, 0xb9, 0xab, 0xbf,
0x54, 0x4a, 0xaf, 0xad, 0x42, 0x5b, 0x7f, 0xbe, 0x3c, 0x82, 0x1b, 0xb2, 0xff, 0xb6, 0x8a, 0xa2,
0xcb, 0x73, 0x5f, 0xae, 0x4d, 0x63, 0xba, 0x36, 0xed, 0x01, 0x58, 0x55, 0x55, 0xd5, 0x05, 0x23,
0x98, 0x3f, 0x60, 0x1e, 0x23, 0x29, 0x23, 0x7e, 0xf1, 0x6c, 0x9e, 0x2a, 0x66, 0xe3, 0xb2, 0xfd,
0x50, 0x6d, 0x87, 0x39, 0xa8, 0x33, 0x96, 0xd7, 0x19, 0xff, 0xe4, 0xb7, 0x80, 0x74, 0x4f, 0xea,
0x0e, 0xde, 0x81, 0x2b, 0x5e, 0x0f, 0x8c, 0x32, 0x2f, 0x94, 0xfb, 0xb7, 0x21, 0xf6, 0xaf, 0x29,
0x28, 0x62, 0x01, 0xcb, 0x15, 0x15, 0x48, 0x6e, 0x53, 0x6e, 0x67, 0x4e, 0x10, 0xcc, 0x15, 0x00,
0xd1, 0x52, 0xb2, 0x1b, 0x5a, 0x52, 0x97, 0x53, 0xb6, 0x38, 0xc1, 0x5e, 0x85, 0xe5, 0x4f, 0x31,
0xe3, 0x2f, 0x89, 0x64, 0x8b, 0xc6, 0x43, 0x32, 0xca, 0x12, 0x4f, 0xbb, 0x0a, 0xfb, 0x47, 0x03,
0x56, 0xce, 0x11, 0x50, 0x01, 0x5b, 0xd0, 0x8e, 0xbc, 0x94, 0xe1, 0x24, 0xef, 0x92, 0xfc, 0x38,
0x9d, 0x8a, 0xda, 0x65, 0xa9, 0xa8, 0x57, 0x52, 0x71, 0x1d, 0x5a, 0x91, 0x77, 0xe6, 0x46, 0x87,
0xea, 0xa9, 0xd0, 0x8c, 0xbc, 0xb3, 0x67, 0x87, 0x0f, 0xfe, 0x68, 0x43, 0xef, 0x00, 0x7b, 0xa7,
0x18, 0x07, 0x02, 0x18, 0x1a, 0xe5, 0x0d, 0x51, 0xfe, 0xd1, 0x85, 0x6e, 0x4f, 0x57, 0xfe, 0xcc,
0x5f, 0x79, 0x83, 0x3b, 0x97, 0x89, 0xa9, 0xda, 0xba, 0x82, 0xf6, 0xa1, 0xab, 0xfd, 0xaa, 0x41,
0xcb, 0x9a, 0x62, 0xe5, 0xc7, 0xda, 0x60, 0xe5, 0x1c, 0xae, 0x6e, 0x4d, 0xdb, 0xce, 0xba, 0xb5,
0xea, 0x7b, 0x40, 0xb7, 0x36, 0x6b, 0xa5, 0x0b, 0x6b, 0xda, 0xe6, 0xd5, 0xad, 0x55, 0x77, 0xbd,
0x6e, 0x6d, 0xd6, 0xba, 0x16, 0xd6, 0xb4, 0xf5, 0xa8, 0x5b, 0xab, 0xae, 0x71, 0xdd, 0xda, 0xac,
0x9d, 0x7a, 0x05, 0x7d, 0x0d, 0xf3, 0x95, 0xc5, 0x85, 0xec, 0x89, 0xd6, 0x79, 0x1b, 0x77, 0xb0,
0x7e, 0xa1, 0x4c, 0x61, 0xff, 0x05, 0xf4, 0xf4, 0x85, 0x82, 0x34, 0x40, 0x33, 0x56, 0xe2, 0x60,
0xf5, 0x3c, 0xb6, 0x6e, 0x50, 0x9f, 0x95, 0xba, 0xc1, 0x19, 0xdb, 0x42, 0x37, 0x38, 0x6b, 0xc4,
0xda, 0x57, 0xd0, 0x57, 0x30, 0x37, 0x3d, 0xb3, 0xd0, 0xad, 0xe9, 0xb4, 0x55, 0x46, 0xe1, 0xc0,
0xbe, 0x48, 0xa4, 0x30, 0xbe, 0x07, 0x30, 0x19, 0x45, 0x68, 0x69, 0xa2, 0x53, 0x19, 0x85, 0x83,
0xe5, 0xd9, 0xcc, 0xc2, 0xd4, 0x37, 0x70, 0x7d, 0x66, 0xbf, 0x23, 0xad, 0x49, 0x2e, 0x9a, 0x18,
0x83, 0xff, 0x5f, 0x2a, 0x97, 0xfb, 0x7a, 0xba, 0x0a, 0x73, 0xa9, 0x6c, 0xe3, 0x61, 0xba, 0xe9,
0x87, 0x04, 0xc7, 0xec, 0x29, 0x08, 0x8d, 0x97, 0x09, 0x65, 0xf4, 0xb0, 0x25, 0xfe, 0xad, 0xf9,
0xe0, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2d, 0xec, 0xb0, 0x56, 0xbc, 0x11, 0x00, 0x00,
}

69
weed/pb/filer_pb/filer_pb_helper.go Normal file
View File

@ -0,0 +1,69 @@
package filer_pb
import (
"github.com/chrislusf/seaweedfs/weed/storage/needle"
)
func toFileIdObject(fileIdStr string) (*FileId, error) {
t, err := needle.ParseFileIdFromString(fileIdStr)
if err != nil {
return nil, err
}
return &FileId{
VolumeId: uint32(t.VolumeId),
Cookie: uint32(t.Cookie),
FileKey: uint64(t.Key),
}, nil
}
func (fid *FileId) toFileIdString() string {
return needle.NewFileId(needle.VolumeId(fid.VolumeId), fid.FileKey, fid.Cookie).String()
}
func (c *FileChunk) GetFileIdString() string {
if c.FileId != "" {
return c.FileId
}
if c.Fid != nil {
c.FileId = c.Fid.toFileIdString()
return c.FileId
}
return ""
}
func BeforeEntrySerialization(chunks []*FileChunk) {
for _, chunk := range chunks {
if chunk.FileId != "" {
if fid, err := toFileIdObject(chunk.FileId); err == nil {
chunk.Fid = fid
chunk.FileId = ""
}
}
if chunk.SourceFileId != "" {
if fid, err := toFileIdObject(chunk.SourceFileId); err == nil {
chunk.SourceFid = fid
chunk.SourceFileId = ""
}
}
}
}
func AfterEntryDeserialization(chunks []*FileChunk) {
for _, chunk := range chunks {
if chunk.Fid != nil && chunk.FileId == "" {
chunk.FileId = chunk.Fid.toFileIdString()
}
if chunk.SourceFid != nil && chunk.SourceFileId == "" {
chunk.SourceFileId = chunk.SourceFid.toFileIdString()
}
}
}

17
weed/pb/filer_pb/filer_pb_helper_test.go Normal file
View File

@ -0,0 +1,17 @@
package filer_pb
import (
"testing"
"github.com/golang/protobuf/proto"
)
func TestFileIdSize(t *testing.T) {
fileIdStr := "11745,0293434534cbb9892b"
fid, _ := toFileIdObject(fileIdStr)
bytes, _ := proto.Marshal(fid)
println(len(fileIdStr))
println(len(bytes))
}

Some files were not shown because too many files have changed in this diff Show More