mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-12-14 10:39:24 +08:00
Merge branch 'chrislusf-master' into a
This commit is contained in:
commit
391170c3f4
10
.github/dependabot.yml
vendored
Normal file
10
.github/dependabot.yml
vendored
Normal file
@ -0,0 +1,10 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
- package-ecosystem: gomod
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: daily
|
14
.github/workflows/binaries_dev.yml
vendored
14
.github/workflows/binaries_dev.yml
vendored
@ -12,7 +12,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Delete old release assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
uses: mknejp/delete-release-assets@a8aaab13272b1eaac16cc46dddd3f725b97ee05a # v1
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
@ -31,13 +31,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@v1.22
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@ -53,7 +53,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@v1.22
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@ -78,13 +78,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@v1.22
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@ -100,7 +100,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@v1.22
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
6
.github/workflows/binaries_release0.yml
vendored
6
.github/workflows/binaries_release0.yml
vendored
@ -23,9 +23,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@v1.22
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@ -39,7 +39,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@v1.22
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
6
.github/workflows/binaries_release1.yml
vendored
6
.github/workflows/binaries_release1.yml
vendored
@ -23,9 +23,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@v1.22
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@ -39,7 +39,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@v1.22
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
6
.github/workflows/binaries_release2.yml
vendored
6
.github/workflows/binaries_release2.yml
vendored
@ -23,9 +23,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@v1.22
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@ -39,7 +39,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@v1.22
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
6
.github/workflows/binaries_release3.yml
vendored
6
.github/workflows/binaries_release3.yml
vendored
@ -23,9 +23,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@v1.22
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@ -39,7 +39,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@v1.22
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
17
.github/workflows/container_dev.yml
vendored
17
.github/workflows/container_dev.yml
vendored
@ -5,6 +5,9 @@ on:
|
||||
branches: [ master ]
|
||||
workflow_dispatch: []
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-dev-containers:
|
||||
@ -13,11 +16,11 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v3
|
||||
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@ -30,30 +33,30 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v1
|
||||
uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v1
|
||||
uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v1
|
||||
uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v2
|
||||
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
17
.github/workflows/container_latest.yml
vendored
17
.github/workflows/container_latest.yml
vendored
@ -6,6 +6,9 @@ on:
|
||||
- '*'
|
||||
workflow_dispatch: []
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-latest-container:
|
||||
@ -14,11 +17,11 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v3
|
||||
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@ -31,30 +34,30 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v1
|
||||
uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v1
|
||||
uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v1
|
||||
uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v2
|
||||
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
15
.github/workflows/container_release1.yml
vendored
15
.github/workflows/container_release1.yml
vendored
@ -6,6 +6,9 @@ on:
|
||||
- '*'
|
||||
workflow_dispatch: []
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-default-release-container:
|
||||
runs-on: [ubuntu-latest]
|
||||
@ -13,11 +16,11 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v3
|
||||
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@ -31,20 +34,20 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v1
|
||||
uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v1
|
||||
uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v2
|
||||
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
15
.github/workflows/container_release2.yml
vendored
15
.github/workflows/container_release2.yml
vendored
@ -6,6 +6,9 @@ on:
|
||||
- '*'
|
||||
workflow_dispatch: []
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-large-release-container:
|
||||
@ -14,11 +17,11 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v3
|
||||
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@ -32,20 +35,20 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v1
|
||||
uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v1
|
||||
uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v2
|
||||
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
15
.github/workflows/container_release3.yml
vendored
15
.github/workflows/container_release3.yml
vendored
@ -6,6 +6,9 @@ on:
|
||||
- '*'
|
||||
workflow_dispatch: []
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-large-release-container_rocksdb:
|
||||
@ -14,11 +17,11 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v3
|
||||
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@ -32,20 +35,20 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v1
|
||||
uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v1
|
||||
uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v2
|
||||
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
7
.github/workflows/go.yml
vendored
7
.github/workflows/go.yml
vendored
@ -10,6 +10,9 @@ concurrency:
|
||||
group: ${{ github.head_ref }}/go
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build:
|
||||
@ -18,13 +21,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v2
|
||||
uses: actions/setup-go@f6164bd8c8acb4a71fb2791a8b6c4024ff038dab # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
|
@ -33,7 +33,7 @@ Your support will be really appreciated by me and other supporters!
|
||||
|
||||
|
||||
### Gold Sponsors
|
||||
![shuguang](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/shuguang.png)
|
||||
- [![nodion](https://www.nodion.com/img/logo.svg)](https://www.nodion.com)
|
||||
|
||||
---
|
||||
|
||||
|
@ -36,7 +36,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: 'volume -dataCenter dc1 -rack v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@ -46,7 +46,7 @@ services:
|
||||
ports:
|
||||
- 8082:8082
|
||||
- 18082:18082
|
||||
command: 'volume -dataCenter dc2 -rack v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@ -56,7 +56,7 @@ services:
|
||||
ports:
|
||||
- 8083:8083
|
||||
- 18083:18083
|
||||
command: 'volume -dataCenter dc3 -rack v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@ -67,7 +67,7 @@ services:
|
||||
- 8888:8888
|
||||
- 18888:18888
|
||||
- 8111:8111
|
||||
command: 'filer -defaultReplicaPlacement 100 -iam -master="master0:9333,master1:9334,master2:9335"'
|
||||
command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335"'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@ -85,4 +85,4 @@ services:
|
||||
- master2
|
||||
- volume1
|
||||
- volume2
|
||||
- filer
|
||||
- filer
|
||||
|
128
go.mod
128
go.mod
@ -3,21 +3,21 @@ module github.com/chrislusf/seaweedfs
|
||||
go 1.18
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.94.1 // indirect
|
||||
cloud.google.com/go/pubsub v1.3.1
|
||||
cloud.google.com/go/storage v1.16.1
|
||||
cloud.google.com/go v0.100.2 // indirect
|
||||
cloud.google.com/go/pubsub v1.19.0
|
||||
cloud.google.com/go/storage v1.21.0
|
||||
github.com/Azure/azure-pipeline-go v0.2.3
|
||||
github.com/Azure/azure-storage-blob-go v0.14.0
|
||||
github.com/OneOfOne/xxhash v1.2.2
|
||||
github.com/Shopify/sarama v1.23.1
|
||||
github.com/aws/aws-sdk-go v1.35.3
|
||||
github.com/aws/aws-sdk-go v1.43.25
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
|
||||
github.com/bwmarrin/snowflake v0.3.0
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
github.com/cespare/xxhash/v2 v2.1.1 // indirect
|
||||
github.com/chrislusf/raft v1.0.7
|
||||
github.com/colinmarc/hdfs/v2 v2.2.0
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/chrislusf/raft v1.0.8
|
||||
github.com/colinmarc/hdfs/v2 v2.3.0
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
@ -32,13 +32,13 @@ require (
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fclairamb/ftpserverlib v0.8.0
|
||||
github.com/fclairamb/ftpserverlib v0.17.0
|
||||
github.com/frankban/quicktest v1.7.2 // indirect
|
||||
github.com/fsnotify/fsnotify v1.4.9 // indirect
|
||||
github.com/fsnotify/fsnotify v1.5.1 // indirect
|
||||
github.com/go-errors/errors v1.1.1 // indirect
|
||||
github.com/go-redis/redis/v8 v8.4.4
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/go-redsync/redsync/v4 v4.4.1
|
||||
github.com/go-sql-driver/mysql v1.5.0
|
||||
github.com/go-sql-driver/mysql v1.6.0
|
||||
github.com/go-stack/stack v1.8.0 // indirect
|
||||
github.com/go-zookeeper/zk v1.0.2 // indirect
|
||||
github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
|
||||
@ -46,12 +46,11 @@ require (
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/btree v1.0.0
|
||||
github.com/google/go-cmp v0.5.6 // indirect
|
||||
github.com/google/btree v1.0.1
|
||||
github.com/google/go-cmp v0.5.7 // indirect
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/google/wire v0.4.0 // indirect
|
||||
github.com/googleapis/gax-go v2.0.2+incompatible // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.1.0 // indirect
|
||||
github.com/google/wire v0.5.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.1.1 // indirect
|
||||
github.com/gorilla/mux v1.7.4
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
|
||||
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
|
||||
@ -60,34 +59,29 @@ require (
|
||||
github.com/hashicorp/go-uuid v1.0.2 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/jcmturner/gofork v1.0.0 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.1
|
||||
github.com/jinzhu/copier v0.2.8
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.2
|
||||
github.com/jinzhu/copier v0.3.5
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.11
|
||||
github.com/karlseguin/ccache/v2 v2.0.7
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/compress v1.13.6 // indirect
|
||||
github.com/klauspost/cpuid v1.2.1 // indirect
|
||||
github.com/klauspost/reedsolomon v1.9.2
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/lib/pq v1.10.0
|
||||
github.com/lib/pq v1.10.4
|
||||
github.com/linxGnu/grocksdb v1.6.38
|
||||
github.com/magiconair/properties v1.8.1 // indirect
|
||||
github.com/mailru/easyjson v0.7.1 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.1 // indirect
|
||||
github.com/mattn/go-isatty v0.0.12 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
|
||||
github.com/mitchellh/mapstructure v1.1.2 // indirect
|
||||
github.com/mitchellh/mapstructure v1.4.1 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.1 // indirect
|
||||
github.com/nats-io/jwt v1.0.1 // indirect
|
||||
github.com/nats-io/nats.go v1.10.0 // indirect
|
||||
github.com/nats-io/nkeys v0.2.0 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/olivere/elastic/v7 v7.0.19
|
||||
github.com/pelletier/go-toml v1.7.0 // indirect
|
||||
github.com/peterh/liner v1.1.0
|
||||
github.com/peterh/liner v1.2.2
|
||||
github.com/pierrec/lz4 v2.2.7+incompatible // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
@ -99,18 +93,18 @@ require (
|
||||
github.com/prometheus/procfs v0.6.0 // indirect
|
||||
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
|
||||
github.com/seaweedfs/goexif v1.0.2
|
||||
github.com/sirupsen/logrus v1.6.0 // indirect
|
||||
github.com/seaweedfs/goexif v2.0.0+incompatible
|
||||
github.com/sirupsen/logrus v1.8.1 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/spf13/afero v1.6.0 // indirect
|
||||
github.com/spf13/afero v1.7.0 // indirect
|
||||
github.com/spf13/cast v1.3.0 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/spf13/viper v1.4.0
|
||||
github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/streadway/amqp v1.0.0
|
||||
github.com/stretchr/testify v1.7.1
|
||||
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
|
||||
github.com/tidwall/gjson v1.10.2
|
||||
github.com/tidwall/gjson v1.14.0
|
||||
github.com/tidwall/match v1.1.1
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
|
||||
@ -124,25 +118,24 @@ require (
|
||||
github.com/xdg-go/stringprep v1.0.2 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.5.0
|
||||
go.mongodb.org/mongo-driver v1.8.0
|
||||
go.mongodb.org/mongo-driver v1.8.4
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
go.opentelemetry.io/otel v0.15.0 // indirect
|
||||
gocloud.dev v0.20.0
|
||||
gocloud.dev v0.24.0
|
||||
gocloud.dev/pubsub/natspubsub v0.20.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.20.0
|
||||
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.24.0
|
||||
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa // indirect
|
||||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
|
||||
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
|
||||
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
|
||||
golang.org/x/sys v0.0.0-20220111092808-5a964db01320
|
||||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
|
||||
golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect
|
||||
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
||||
google.golang.org/api v0.57.0
|
||||
google.golang.org/api v0.73.0
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6 // indirect
|
||||
google.golang.org/grpc v1.40.0
|
||||
google.golang.org/protobuf v1.27.1
|
||||
google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6 // indirect
|
||||
google.golang.org/grpc v1.45.0
|
||||
google.golang.org/protobuf v1.28.0
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
|
||||
gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
|
||||
@ -150,14 +143,14 @@ require (
|
||||
gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
|
||||
gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/cc/v3 v3.33.5 // indirect
|
||||
modernc.org/ccgo/v3 v3.9.4 // indirect
|
||||
modernc.org/libc v1.9.5 // indirect
|
||||
modernc.org/mathutil v1.2.2 // indirect
|
||||
modernc.org/memory v1.0.4 // indirect
|
||||
modernc.org/cc/v3 v3.35.24 // indirect
|
||||
modernc.org/ccgo/v3 v3.15.17 // indirect
|
||||
modernc.org/libc v1.14.12 // indirect
|
||||
modernc.org/mathutil v1.4.1 // indirect
|
||||
modernc.org/memory v1.0.7 // indirect
|
||||
modernc.org/opt v0.1.1 // indirect
|
||||
modernc.org/sqlite v1.10.7
|
||||
modernc.org/strutil v1.1.0 // indirect
|
||||
modernc.org/sqlite v1.15.3
|
||||
modernc.org/strutil v1.1.1 // indirect
|
||||
modernc.org/token v1.0.0 // indirect
|
||||
)
|
||||
|
||||
@ -167,30 +160,47 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/kms v1.0.0 // indirect
|
||||
cloud.google.com/go/compute v1.5.0 // indirect
|
||||
cloud.google.com/go/iam v0.1.1 // indirect
|
||||
github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect
|
||||
github.com/arangodb/go-driver v1.2.1 // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.9.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.7.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.4.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 // indirect
|
||||
github.com/aws/smithy-go v1.8.0 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/fclairamb/go-log v0.1.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 // indirect
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.2 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.7 // indirect
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
|
||||
github.com/nats-io/nats.go v1.11.0 // indirect
|
||||
github.com/nats-io/nkeys v0.3.0 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/philhofer/fwd v1.1.1 // indirect
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/tinylib/msgp v1.1.6 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.5.0 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.7.0 // indirect
|
||||
go.uber.org/zap v1.17.0 // indirect
|
||||
golang.org/x/mod v0.4.2 // indirect
|
||||
go.uber.org/zap v1.19.0 // indirect
|
||||
golang.org/x/mod v0.5.0 // indirect
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
|
||||
lukechampine.com/uint128 v1.1.1 // indirect
|
||||
)
|
||||
|
||||
// replace github.com/chrislusf/raft => /Users/chrislu/go/src/github.com/chrislusf/raft
|
||||
|
@ -1,5 +1,5 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "2.94"
|
||||
version: "2.94"
|
||||
appVersion: "2.96"
|
||||
version: "2.96"
|
||||
|
@ -129,7 +129,7 @@ func runBenchmark(cmd *Command, args []string) bool {
|
||||
defer pprof.StopCPUProfile()
|
||||
}
|
||||
|
||||
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", "", pb.ServerAddresses(*b.masters).ToAddresses())
|
||||
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", "", pb.ServerAddresses(*b.masters).ToAddressMap())
|
||||
go b.masterClient.KeepConnectedToMaster()
|
||||
b.masterClient.WaitUntilConnected()
|
||||
|
||||
|
@ -13,7 +13,7 @@ import (
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/server"
|
||||
weed_server "github.com/chrislusf/seaweedfs/weed/server"
|
||||
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
)
|
||||
@ -29,7 +29,7 @@ var (
|
||||
)
|
||||
|
||||
type FilerOptions struct {
|
||||
masters []pb.ServerAddress
|
||||
masters map[string]pb.ServerAddress
|
||||
mastersString *string
|
||||
ip *string
|
||||
bindIp *string
|
||||
@ -103,6 +103,7 @@ func init() {
|
||||
|
||||
// start iam on filer
|
||||
filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
|
||||
filerIamOptions.ip = cmdFiler.Flag.String("iam.ip", *f.ip, "iam server http listen ip address")
|
||||
filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
|
||||
}
|
||||
|
||||
@ -170,7 +171,7 @@ func runFiler(cmd *Command, args []string) bool {
|
||||
}()
|
||||
}
|
||||
|
||||
f.masters = pb.ServerAddresses(*f.mastersString).ToAddresses()
|
||||
f.masters = pb.ServerAddresses(*f.mastersString).ToAddressMap()
|
||||
|
||||
f.startFiler()
|
||||
|
||||
|
@ -67,7 +67,7 @@ func (iamopt *IamOptions) startIamServer() bool {
|
||||
}
|
||||
}
|
||||
|
||||
masters := pb.ServerAddresses(*iamopt.masters).ToAddresses()
|
||||
masters := pb.ServerAddresses(*iamopt.masters).ToAddressMap()
|
||||
router := mux.NewRouter().SkipClean(true)
|
||||
_, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{
|
||||
Masters: masters,
|
||||
|
@ -127,13 +127,18 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
*masterOption.portGrpc = 10000 + *masterOption.port
|
||||
}
|
||||
if *masterOption.ipBind == "" {
|
||||
*masterOption.ipBind = "localhost"
|
||||
*masterOption.ipBind = *masterOption.ip
|
||||
}
|
||||
|
||||
myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.portGrpc, *masterOption.peers)
|
||||
|
||||
masterPeers := make(map[string]pb.ServerAddress)
|
||||
for _, peer := range peers {
|
||||
masterPeers[peer.String()] = peer
|
||||
}
|
||||
|
||||
r := mux.NewRouter()
|
||||
ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), peers)
|
||||
ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), masterPeers)
|
||||
listeningAddress := util.JoinHostPort(*masterOption.ipBind, *masterOption.port)
|
||||
glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
|
||||
masterListener, masterLocalListner, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0)
|
||||
@ -144,7 +149,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
// start raftServer
|
||||
raftServerOption := &weed_server.RaftServerOption{
|
||||
GrpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.master"),
|
||||
Peers: peers,
|
||||
Peers: masterPeers,
|
||||
ServerAddr: myMasterAddress,
|
||||
DataDir: util.ResolvePath(*masterOption.metaFolder),
|
||||
Topo: ms.Topo,
|
||||
|
@ -3,17 +3,18 @@ package command
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/server"
|
||||
weed_server "github.com/chrislusf/seaweedfs/weed/server"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/gorilla/mux"
|
||||
"google.golang.org/grpc/reflection"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -45,13 +46,13 @@ var cmdMasterFollower = &Command{
|
||||
Short: "start a master follower",
|
||||
Long: `start a master follower to provide volume=>location mapping service
|
||||
|
||||
The master follower does not participate in master election.
|
||||
The master follower does not participate in master election.
|
||||
It just follow the existing masters, and listen for any volume location changes.
|
||||
|
||||
In most cases, the master follower is not needed. In big data centers with thousands of volume
|
||||
servers. In theory, the master may have trouble to keep up with the write requests and read requests.
|
||||
|
||||
The master follower can relieve the master from from read requests, which only needs to
|
||||
The master follower can relieve the master from from read requests, which only needs to
|
||||
lookup a fileId or volumeId.
|
||||
|
||||
The master follower currently can handle fileId lookup requests:
|
||||
@ -82,7 +83,7 @@ func runMasterFollower(cmd *Command, args []string) bool {
|
||||
func startMasterFollower(masterOptions MasterOptions) {
|
||||
|
||||
// collect settings from main masters
|
||||
masters := pb.ServerAddresses(*mf.peers).ToAddresses()
|
||||
masters := pb.ServerAddresses(*mf.peers).ToAddressMap()
|
||||
|
||||
var err error
|
||||
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.master")
|
||||
@ -112,7 +113,7 @@ func startMasterFollower(masterOptions MasterOptions) {
|
||||
option.IsFollower = true
|
||||
|
||||
if *masterOptions.ipBind == "" {
|
||||
*masterOptions.ipBind = "localhost"
|
||||
*masterOptions.ipBind = *masterOptions.ip
|
||||
}
|
||||
|
||||
r := mux.NewRouter()
|
||||
|
@ -191,7 +191,7 @@ func runServer(cmd *Command, args []string) bool {
|
||||
// ip address
|
||||
masterOptions.ip = serverIp
|
||||
masterOptions.ipBind = serverBindIp
|
||||
filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
|
||||
filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddressMap()
|
||||
filerOptions.ip = serverIp
|
||||
filerOptions.bindIp = serverBindIp
|
||||
s3Options.bindIp = serverBindIp
|
||||
|
@ -23,7 +23,13 @@ func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
|
||||
}
|
||||
|
||||
func FileSize(entry *filer_pb.Entry) (size uint64) {
|
||||
return maxUint64(TotalSize(entry.Chunks), entry.Attributes.FileSize)
|
||||
fileSize := entry.Attributes.FileSize
|
||||
if entry.RemoteEntry != nil {
|
||||
if entry.RemoteEntry.RemoteMtime > entry.Attributes.Mtime {
|
||||
fileSize = maxUint64(fileSize, uint64(entry.RemoteEntry.RemoteSize))
|
||||
}
|
||||
}
|
||||
return maxUint64(TotalSize(entry.Chunks), fileSize)
|
||||
}
|
||||
|
||||
func ETag(entry *filer_pb.Entry) (etag string) {
|
||||
|
@ -49,7 +49,7 @@ type Filer struct {
|
||||
UniqueFileId uint32
|
||||
}
|
||||
|
||||
func NewFiler(masters []pb.ServerAddress, grpcDialOption grpc.DialOption,
|
||||
func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption,
|
||||
filerHost pb.ServerAddress, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
|
||||
f := &Filer{
|
||||
MasterClient: wdclient.NewMasterClient(grpcDialOption, cluster.FilerType, filerHost, dataCenter, masters),
|
||||
|
@ -76,9 +76,6 @@ func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) (no
|
||||
}
|
||||
} else {
|
||||
if _, found := ma.peerStatues[address]; found {
|
||||
ma.peerStatues[address] -= 1
|
||||
}
|
||||
if ma.peerStatues[address] <= 0 {
|
||||
delete(ma.peerStatues, address)
|
||||
}
|
||||
}
|
||||
|
@ -62,6 +62,7 @@ func SaveInsideFiler(client filer_pb.SeaweedFilerClient, dir, name string, conte
|
||||
},
|
||||
Content: content,
|
||||
},
|
||||
SkipCheckParentDirectory: true,
|
||||
})
|
||||
} else if err == nil {
|
||||
entry := resp.Entry
|
||||
|
@ -33,7 +33,7 @@ type IamS3ApiConfigure struct {
|
||||
}
|
||||
|
||||
type IamServerOption struct {
|
||||
Masters []pb.ServerAddress
|
||||
Masters map[string]pb.ServerAddress
|
||||
Filer pb.ServerAddress
|
||||
Port int
|
||||
GrpcDialOption grpc.DialOption
|
||||
|
@ -77,9 +77,10 @@ func (wfs *WFS) Mknod(cancel <-chan struct{}, in *fuse.MknodIn, name string, out
|
||||
defer wfs.mapPbIdFromFilerToLocal(newEntry)
|
||||
|
||||
request := &filer_pb.CreateEntryRequest{
|
||||
Directory: string(dirFullPath),
|
||||
Entry: newEntry,
|
||||
Signatures: []int32{wfs.signature},
|
||||
Directory: string(dirFullPath),
|
||||
Entry: newEntry,
|
||||
Signatures: []int32{wfs.signature},
|
||||
SkipCheckParentDirectory: true,
|
||||
}
|
||||
|
||||
glog.V(1).Infof("mknod: %v", request)
|
||||
|
@ -140,9 +140,10 @@ func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status {
|
||||
}
|
||||
|
||||
request := &filer_pb.CreateEntryRequest{
|
||||
Directory: string(dir),
|
||||
Entry: entry,
|
||||
Signatures: []int32{wfs.signature},
|
||||
Directory: string(dir),
|
||||
Entry: entry,
|
||||
Signatures: []int32{wfs.signature},
|
||||
SkipCheckParentDirectory: true,
|
||||
}
|
||||
|
||||
glog.V(4).Infof("%s set chunks: %v", fileFullPath, len(entry.Chunks))
|
||||
|
@ -206,7 +206,7 @@ func WithMasterClient(streamingMode bool, master ServerAddress, grpcDialOption g
|
||||
|
||||
}
|
||||
|
||||
func WithOneOfGrpcMasterClients(streamingMode bool, masterGrpcAddresses []ServerAddress, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) (err error) {
|
||||
func WithOneOfGrpcMasterClients(streamingMode bool, masterGrpcAddresses map[string]ServerAddress, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) (err error) {
|
||||
|
||||
for _, masterGrpcAddress := range masterGrpcAddresses {
|
||||
err = WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
|
||||
|
@ -62,6 +62,10 @@ message RemoteConf {
|
||||
string storj_secret_key = 66;
|
||||
string storj_endpoint = 67;
|
||||
|
||||
string contabo_access_key = 68;
|
||||
string contabo_secret_key = 69;
|
||||
string contabo_endpoint = 70;
|
||||
string contabo_region = 71;
|
||||
}
|
||||
|
||||
message RemoteStorageMapping {
|
||||
|
@ -1,7 +1,7 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.26.0
|
||||
// protoc v3.17.3
|
||||
// protoc v3.6.1
|
||||
// source: remote.proto
|
||||
|
||||
package remote_pb
|
||||
@ -69,6 +69,10 @@ type RemoteConf struct {
|
||||
StorjAccessKey string `protobuf:"bytes,65,opt,name=storj_access_key,json=storjAccessKey,proto3" json:"storj_access_key,omitempty"`
|
||||
StorjSecretKey string `protobuf:"bytes,66,opt,name=storj_secret_key,json=storjSecretKey,proto3" json:"storj_secret_key,omitempty"`
|
||||
StorjEndpoint string `protobuf:"bytes,67,opt,name=storj_endpoint,json=storjEndpoint,proto3" json:"storj_endpoint,omitempty"`
|
||||
ContaboAccessKey string `protobuf:"bytes,68,opt,name=contabo_access_key,json=contaboAccessKey,proto3" json:"contabo_access_key,omitempty"`
|
||||
ContaboSecretKey string `protobuf:"bytes,69,opt,name=contabo_secret_key,json=contaboSecretKey,proto3" json:"contabo_secret_key,omitempty"`
|
||||
ContaboEndpoint string `protobuf:"bytes,70,opt,name=contabo_endpoint,json=contaboEndpoint,proto3" json:"contabo_endpoint,omitempty"`
|
||||
ContaboRegion string `protobuf:"bytes,71,opt,name=contabo_region,json=contaboRegion,proto3" json:"contabo_region,omitempty"`
|
||||
}
|
||||
|
||||
func (x *RemoteConf) Reset() {
|
||||
@ -390,6 +394,34 @@ func (x *RemoteConf) GetStorjEndpoint() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *RemoteConf) GetContaboAccessKey() string {
|
||||
if x != nil {
|
||||
return x.ContaboAccessKey
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *RemoteConf) GetContaboSecretKey() string {
|
||||
if x != nil {
|
||||
return x.ContaboSecretKey
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *RemoteConf) GetContaboEndpoint() string {
|
||||
if x != nil {
|
||||
return x.ContaboEndpoint
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *RemoteConf) GetContaboRegion() string {
|
||||
if x != nil {
|
||||
return x.ContaboRegion
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type RemoteStorageMapping struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
@ -512,7 +544,7 @@ var File_remote_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_remote_proto_rawDesc = []byte{
|
||||
0x0a, 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09,
|
||||
0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x22, 0xe2, 0x0d, 0x0a, 0x0a, 0x52, 0x65,
|
||||
0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x22, 0x90, 0x0f, 0x0a, 0x0a, 0x52, 0x65,
|
||||
0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65,
|
||||
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04,
|
||||
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65,
|
||||
@ -622,35 +654,46 @@ var file_remote_proto_rawDesc = []byte{
|
||||
0x79, 0x18, 0x42, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x53, 0x65,
|
||||
0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x6a,
|
||||
0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x43, 0x20, 0x01, 0x28, 0x09, 0x52,
|
||||
0x0d, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x22, 0xff,
|
||||
0x01, 0x0a, 0x14, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65,
|
||||
0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69,
|
||||
0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x6d, 0x6f,
|
||||
0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72,
|
||||
0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x70, 0x70, 0x69,
|
||||
0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e,
|
||||
0x67, 0x73, 0x12, 0x3d, 0x0a, 0x1b, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x62, 0x75,
|
||||
0x63, 0x6b, 0x65, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d,
|
||||
0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79,
|
||||
0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d,
|
||||
0x65, 0x1a, 0x5d, 0x0a, 0x0d, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74,
|
||||
0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
|
||||
0x03, 0x6b, 0x65, 0x79, 0x12, 0x36, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
|
||||
0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e,
|
||||
0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63,
|
||||
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01,
|
||||
0x22, 0x57, 0x0a, 0x15, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67,
|
||||
0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d,
|
||||
0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a,
|
||||
0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62,
|
||||
0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20,
|
||||
0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x42, 0x50, 0x0a, 0x10, 0x73, 0x65, 0x61,
|
||||
0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46,
|
||||
0x69, 0x6c, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75,
|
||||
0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f,
|
||||
0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70,
|
||||
0x62, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f,
|
||||
0x74, 0x6f, 0x33,
|
||||
0x0d, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2c,
|
||||
0x0a, 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73,
|
||||
0x5f, 0x6b, 0x65, 0x79, 0x18, 0x44, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x74,
|
||||
0x61, 0x62, 0x6f, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x12,
|
||||
0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b,
|
||||
0x65, 0x79, 0x18, 0x45, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62,
|
||||
0x6f, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6f,
|
||||
0x6e, 0x74, 0x61, 0x62, 0x6f, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x46,
|
||||
0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x45, 0x6e, 0x64,
|
||||
0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f,
|
||||
0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x47, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63,
|
||||
0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x22, 0xff, 0x01, 0x0a,
|
||||
0x14, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61,
|
||||
0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67,
|
||||
0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65,
|
||||
0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67,
|
||||
0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67,
|
||||
0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73,
|
||||
0x12, 0x3d, 0x0a, 0x1b, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x62, 0x75, 0x63, 0x6b,
|
||||
0x65, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18,
|
||||
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x42, 0x75,
|
||||
0x63, 0x6b, 0x65, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x1a,
|
||||
0x5d, 0x0a, 0x0d, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79,
|
||||
0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b,
|
||||
0x65, 0x79, 0x12, 0x36, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
|
||||
0x0b, 0x32, 0x20, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65,
|
||||
0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74,
|
||||
0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57,
|
||||
0x0a, 0x15, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c,
|
||||
0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
|
||||
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x62,
|
||||
0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63,
|
||||
0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28,
|
||||
0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x42, 0x50, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65,
|
||||
0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c,
|
||||
0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
|
||||
0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65,
|
||||
0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f,
|
||||
0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
|
||||
0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
@ -86,6 +86,14 @@ func (sa ServerAddresses) ToAddresses() (addresses []ServerAddress) {
|
||||
return
|
||||
}
|
||||
|
||||
func (sa ServerAddresses) ToAddressMap() (addresses map[string]ServerAddress) {
|
||||
addresses = make(map[string]ServerAddress)
|
||||
for _, address := range sa.ToAddresses() {
|
||||
addresses[address.String()] = address
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (sa ServerAddresses) ToAddressStrings() (addresses []string) {
|
||||
parts := strings.Split(string(sa), ",")
|
||||
for _, address := range parts {
|
||||
@ -101,6 +109,13 @@ func ToAddressStrings(addresses []ServerAddress) []string {
|
||||
}
|
||||
return strings
|
||||
}
|
||||
func ToAddressStringsFromMap(addresses map[string]ServerAddress) []string {
|
||||
var strings []string
|
||||
for _, addr := range addresses {
|
||||
strings = append(strings, string(addr))
|
||||
}
|
||||
return strings
|
||||
}
|
||||
func FromAddressStrings(strings []string) []ServerAddress {
|
||||
var addresses []ServerAddress
|
||||
for _, addr := range strings {
|
||||
|
50
weed/remote_storage/s3/contabo.go
Normal file
50
weed/remote_storage/s3/contabo.go
Normal file
@ -0,0 +1,50 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/remote_storage"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func init() {
|
||||
remote_storage.RemoteStorageClientMakers["contabo"] = new(ContaboRemoteStorageMaker)
|
||||
}
|
||||
|
||||
type ContaboRemoteStorageMaker struct{}
|
||||
|
||||
func (s ContaboRemoteStorageMaker) HasBucket() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (s ContaboRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
|
||||
client := &s3RemoteStorageClient{
|
||||
conf: conf,
|
||||
}
|
||||
accessKey := util.Nvl(conf.ContaboAccessKey, os.Getenv("ACCESS_KEY"))
|
||||
secretKey := util.Nvl(conf.ContaboSecretKey, os.Getenv("SECRET_KEY"))
|
||||
|
||||
config := &aws.Config{
|
||||
Endpoint: aws.String(conf.ContaboEndpoint),
|
||||
Region: aws.String(conf.ContaboRegion),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
S3DisableContentMD5Validation: aws.Bool(true),
|
||||
}
|
||||
if accessKey != "" && secretKey != "" {
|
||||
config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "")
|
||||
}
|
||||
|
||||
sess, err := session.NewSession(config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create contabo session: %v", err)
|
||||
}
|
||||
sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
|
||||
client.conn = s3.New(sess)
|
||||
return client, nil
|
||||
}
|
@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@ -62,10 +63,15 @@ type CompleteMultipartUploadResult struct {
|
||||
s3.CompleteMultipartUploadOutput
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput) (output *CompleteMultipartUploadResult, code s3err.ErrorCode) {
|
||||
func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput, parts *CompleteMultipartUpload) (output *CompleteMultipartUploadResult, code s3err.ErrorCode) {
|
||||
|
||||
glog.V(2).Infof("completeMultipartUpload input %v", input)
|
||||
|
||||
completedParts := parts.Parts
|
||||
sort.Slice(completedParts, func(i, j int) bool {
|
||||
return completedParts[i].PartNumber < completedParts[j].PartNumber
|
||||
})
|
||||
|
||||
uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId
|
||||
|
||||
entries, _, err := s3a.list(uploadDirectory, "", "", false, maxPartsList)
|
||||
@ -80,14 +86,16 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
|
||||
return nil, s3err.ErrNoSuchUpload
|
||||
}
|
||||
|
||||
mime := pentry.Attributes.Mime
|
||||
|
||||
var finalParts []*filer_pb.FileChunk
|
||||
var offset int64
|
||||
var mime string
|
||||
|
||||
for _, entry := range entries {
|
||||
if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory {
|
||||
if entry.Name == "0001.part" && entry.Attributes.Mime != "" {
|
||||
mime = entry.Attributes.Mime
|
||||
_, found := findByPartNumber(entry.Name, completedParts)
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
for _, chunk := range entry.Chunks {
|
||||
p := &filer_pb.FileChunk{
|
||||
@ -156,6 +164,20 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
|
||||
return
|
||||
}
|
||||
|
||||
func findByPartNumber(fileName string, parts []CompletedPart) (etag string, found bool) {
|
||||
partNumber, formatErr := strconv.Atoi(fileName[:4])
|
||||
if formatErr != nil {
|
||||
return
|
||||
}
|
||||
x := sort.Search(len(parts), func(i int) bool {
|
||||
return parts[i].PartNumber >= partNumber
|
||||
})
|
||||
if parts[x].PartNumber != partNumber {
|
||||
return
|
||||
}
|
||||
return parts[x].ETag, true
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code s3err.ErrorCode) {
|
||||
|
||||
glog.V(2).Infof("abortMultipartUpload input %v", input)
|
||||
|
@ -4,6 +4,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@ -48,3 +49,85 @@ func TestListPartsResult(t *testing.T) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func Test_findByPartNumber(t *testing.T) {
|
||||
type args struct {
|
||||
fileName string
|
||||
parts []CompletedPart
|
||||
}
|
||||
|
||||
parts := []CompletedPart{
|
||||
CompletedPart{
|
||||
ETag: "xxx",
|
||||
PartNumber: 1,
|
||||
},
|
||||
CompletedPart{
|
||||
ETag: "yyy",
|
||||
PartNumber: 3,
|
||||
},
|
||||
CompletedPart{
|
||||
ETag: "zzz",
|
||||
PartNumber: 5,
|
||||
},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
wantEtag string
|
||||
wantFound bool
|
||||
}{
|
||||
{
|
||||
"first",
|
||||
args{
|
||||
"0001.part",
|
||||
parts,
|
||||
},
|
||||
"xxx",
|
||||
true,
|
||||
},
|
||||
{
|
||||
"second",
|
||||
args{
|
||||
"0002.part",
|
||||
parts,
|
||||
},
|
||||
"",
|
||||
false,
|
||||
},
|
||||
{
|
||||
"third",
|
||||
args{
|
||||
"0003.part",
|
||||
parts,
|
||||
},
|
||||
"yyy",
|
||||
true,
|
||||
},
|
||||
{
|
||||
"fourth",
|
||||
args{
|
||||
"0004.part",
|
||||
parts,
|
||||
},
|
||||
"",
|
||||
false,
|
||||
},
|
||||
{
|
||||
"fifth",
|
||||
args{
|
||||
"0005.part",
|
||||
parts,
|
||||
},
|
||||
"zzz",
|
||||
true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotEtag, gotFound := findByPartNumber(tt.args.fileName, tt.args.parts)
|
||||
assert.Equalf(t, tt.wantEtag, gotEtag, "findByPartNumber(%v, %v)", tt.args.fileName, tt.args.parts)
|
||||
assert.Equalf(t, tt.wantFound, gotFound, "findByPartNumber(%v, %v)", tt.args.fileName, tt.args.parts)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
@ -55,10 +55,11 @@ func (s3a *S3ApiServer) rm(parentDirectoryPath, entryName string, isDeleteData,
|
||||
|
||||
func doDeleteEntry(client filer_pb.SeaweedFilerClient, parentDirectoryPath string, entryName string, isDeleteData bool, isRecursive bool) error {
|
||||
request := &filer_pb.DeleteEntryRequest{
|
||||
Directory: parentDirectoryPath,
|
||||
Name: entryName,
|
||||
IsDeleteData: isDeleteData,
|
||||
IsRecursive: isRecursive,
|
||||
Directory: parentDirectoryPath,
|
||||
Name: entryName,
|
||||
IsDeleteData: isDeleteData,
|
||||
IsRecursive: isRecursive,
|
||||
IgnoreRecursiveError: true,
|
||||
}
|
||||
|
||||
glog.V(1).Infof("delete entry %v/%v: %v", parentDirectoryPath, entryName, request)
|
||||
|
@ -1,11 +1,13 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
|
||||
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
|
||||
weed_server "github.com/chrislusf/seaweedfs/weed/server"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
@ -56,8 +58,16 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http
|
||||
|
||||
// CompleteMultipartUploadHandler - Completes multipart upload.
|
||||
func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
|
||||
|
||||
bucket, object := xhttp.GetBucketAndObject(r)
|
||||
|
||||
parts := &CompleteMultipartUpload{}
|
||||
if err := xmlDecoder(r.Body, parts, r.ContentLength); err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML)
|
||||
return
|
||||
}
|
||||
|
||||
// Get upload id.
|
||||
uploadID, _, _, _ := getObjectResources(r.URL.Query())
|
||||
|
||||
@ -65,7 +75,7 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
|
||||
Bucket: aws.String(bucket),
|
||||
Key: objectKey(aws.String(object)),
|
||||
UploadId: aws.String(uploadID),
|
||||
})
|
||||
}, parts)
|
||||
|
||||
glog.V(2).Info("CompleteMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode)
|
||||
|
||||
@ -268,8 +278,24 @@ func getObjectResources(values url.Values) (uploadID string, partNumberMarker, m
|
||||
return
|
||||
}
|
||||
|
||||
type byCompletedPartNumber []*s3.CompletedPart
|
||||
func xmlDecoder(body io.Reader, v interface{}, size int64) error {
|
||||
var lbody io.Reader
|
||||
if size > 0 {
|
||||
lbody = io.LimitReader(body, size)
|
||||
} else {
|
||||
lbody = body
|
||||
}
|
||||
d := xml.NewDecoder(lbody)
|
||||
d.CharsetReader = func(label string, input io.Reader) (io.Reader, error) {
|
||||
return input, nil
|
||||
}
|
||||
return d.Decode(v)
|
||||
}
|
||||
|
||||
func (a byCompletedPartNumber) Len() int { return len(a) }
|
||||
func (a byCompletedPartNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a byCompletedPartNumber) Less(i, j int) bool { return *a[i].PartNumber < *a[j].PartNumber }
|
||||
type CompleteMultipartUpload struct {
|
||||
Parts []CompletedPart `xml:"Part"`
|
||||
}
|
||||
type CompletedPart struct {
|
||||
ETag string
|
||||
PartNumber int
|
||||
}
|
||||
|
@ -393,7 +393,7 @@ func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb.
|
||||
clusterId, _ := fs.filer.Store.KvGet(context.Background(), []byte("clusterId"))
|
||||
|
||||
t := &filer_pb.GetFilerConfigurationResponse{
|
||||
Masters: pb.ToAddressStrings(fs.option.Masters),
|
||||
Masters: pb.ToAddressStringsFromMap(fs.option.Masters),
|
||||
Collection: fs.option.Collection,
|
||||
Replication: fs.option.DefaultReplication,
|
||||
MaxMb: uint32(fs.option.MaxMB),
|
||||
|
@ -49,7 +49,7 @@ import (
|
||||
)
|
||||
|
||||
type FilerOption struct {
|
||||
Masters []pb.ServerAddress
|
||||
Masters map[string]pb.ServerAddress
|
||||
Collection string
|
||||
DefaultReplication string
|
||||
DisableDirListing bool
|
||||
|
@ -82,7 +82,9 @@ func (fs *FilerServer) DeleteTaggingHandler(w http.ResponseWriter, r *http.Reque
|
||||
toDelete := strings.Split(r.URL.Query().Get("tagging"), ",")
|
||||
deletions := make(map[string]struct{})
|
||||
for _, deletion := range toDelete {
|
||||
deletions[deletion] = struct{}{}
|
||||
if deletion != "" {
|
||||
deletions[deletion] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// delete all tags or specific tags
|
||||
|
@ -131,7 +131,7 @@ func isAppend(r *http.Request) bool {
|
||||
}
|
||||
|
||||
func skipCheckParentDirEntry(r *http.Request) bool {
|
||||
return r.URL.Query().Get("skipCheckParentDir") != "true"
|
||||
return r.URL.Query().Get("skipCheckParentDir") == "true"
|
||||
}
|
||||
|
||||
func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *operation.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64, content []byte) (filerResult *FilerPostResult, replyerr error) {
|
||||
|
@ -75,7 +75,7 @@ type MasterServer struct {
|
||||
Cluster *cluster.Cluster
|
||||
}
|
||||
|
||||
func NewMasterServer(r *mux.Router, option *MasterOption, peers []pb.ServerAddress) *MasterServer {
|
||||
func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.ServerAddress) *MasterServer {
|
||||
|
||||
v := util.GetViper()
|
||||
signingKey := v.GetString("jwt.signing.key")
|
||||
|
@ -5,8 +5,6 @@ import (
|
||||
"math/rand"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
@ -21,7 +19,7 @@ import (
|
||||
|
||||
type RaftServerOption struct {
|
||||
GrpcDialOption grpc.DialOption
|
||||
Peers []pb.ServerAddress
|
||||
Peers map[string]pb.ServerAddress
|
||||
ServerAddr pb.ServerAddress
|
||||
DataDir string
|
||||
Topo *topology.Topology
|
||||
@ -31,7 +29,7 @@ type RaftServerOption struct {
|
||||
}
|
||||
|
||||
type RaftServer struct {
|
||||
peers []pb.ServerAddress // initial peers to join with
|
||||
peers map[string]pb.ServerAddress // initial peers to join with
|
||||
raftServer raft.Server
|
||||
dataDir string
|
||||
serverAddr pb.ServerAddress
|
||||
@ -81,10 +79,11 @@ func NewRaftServer(option *RaftServerOption) (*RaftServer, error) {
|
||||
transporter := raft.NewGrpcTransporter(option.GrpcDialOption)
|
||||
glog.V(0).Infof("Starting RaftServer with %v", option.ServerAddr)
|
||||
|
||||
// always clear previous log to avoid server is promotable
|
||||
os.RemoveAll(path.Join(s.dataDir, "log"))
|
||||
if !option.RaftResumeState {
|
||||
// always clear previous metadata
|
||||
os.RemoveAll(path.Join(s.dataDir, "conf"))
|
||||
os.RemoveAll(path.Join(s.dataDir, "log"))
|
||||
os.RemoveAll(path.Join(s.dataDir, "snapshot"))
|
||||
}
|
||||
if err := os.MkdirAll(path.Join(s.dataDir, "snapshot"), 0600); err != nil {
|
||||
@ -107,39 +106,26 @@ func NewRaftServer(option *RaftServerOption) (*RaftServer, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, peer := range s.peers {
|
||||
if err := s.raftServer.AddPeer(string(peer), peer.ToGrpcAddress()); err != nil {
|
||||
for name, peer := range s.peers {
|
||||
if err := s.raftServer.AddPeer(name, peer.ToGrpcAddress()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Remove deleted peers
|
||||
for existsPeerName := range s.raftServer.Peers() {
|
||||
exists := false
|
||||
var existingPeer pb.ServerAddress
|
||||
for _, peer := range s.peers {
|
||||
if peer.ToGrpcAddress() == existsPeerName {
|
||||
exists, existingPeer = true, peer
|
||||
break
|
||||
}
|
||||
}
|
||||
if exists {
|
||||
if existingPeer, found := s.peers[existsPeerName]; !found {
|
||||
if err := s.raftServer.RemovePeer(existsPeerName); err != nil {
|
||||
glog.V(0).Infoln(err)
|
||||
return nil, err
|
||||
} else {
|
||||
glog.V(0).Infof("removing old peer %s", existingPeer)
|
||||
glog.V(0).Infof("removing old peer: %s", existingPeer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
s.GrpcServer = raft.NewGrpcServer(s.raftServer)
|
||||
|
||||
if s.raftServer.IsLogEmpty() && isTheFirstOne(option.ServerAddr, s.peers) {
|
||||
// Initialize the server by joining itself.
|
||||
// s.DoJoinCommand()
|
||||
}
|
||||
|
||||
glog.V(0).Infof("current cluster leader: %v", s.raftServer.Leader())
|
||||
|
||||
return s, nil
|
||||
@ -155,16 +141,6 @@ func (s *RaftServer) Peers() (members []string) {
|
||||
return
|
||||
}
|
||||
|
||||
func isTheFirstOne(self pb.ServerAddress, peers []pb.ServerAddress) bool {
|
||||
sort.Slice(peers, func(i, j int) bool {
|
||||
return strings.Compare(string(peers[i]), string(peers[j])) < 0
|
||||
})
|
||||
if len(peers) <= 0 {
|
||||
return true
|
||||
}
|
||||
return self == peers[0]
|
||||
}
|
||||
|
||||
func (s *RaftServer) DoJoinCommand() {
|
||||
|
||||
glog.V(0).Infoln("Initializing new cluster")
|
||||
|
@ -23,7 +23,6 @@ type VolumeServer struct {
|
||||
inFlightDownloadDataSize int64
|
||||
concurrentUploadLimit int64
|
||||
concurrentDownloadLimit int64
|
||||
inFlightUploadDataLimitCond *sync.Cond
|
||||
inFlightDownloadDataLimitCond *sync.Cond
|
||||
|
||||
SeedMasterNodes []pb.ServerAddress
|
||||
@ -84,7 +83,6 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
|
||||
fileSizeLimitBytes: int64(fileSizeLimitMB) * 1024 * 1024,
|
||||
isHeartbeating: true,
|
||||
stopChan: make(chan bool),
|
||||
inFlightUploadDataLimitCond: sync.NewCond(new(sync.Mutex)),
|
||||
inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)),
|
||||
concurrentUploadLimit: concurrentUploadLimit,
|
||||
concurrentDownloadLimit: concurrentDownloadLimit,
|
||||
|
@ -1,6 +1,7 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
@ -39,8 +40,14 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
|
||||
stats.ReadRequest()
|
||||
vs.inFlightDownloadDataLimitCond.L.Lock()
|
||||
for vs.concurrentDownloadLimit != 0 && atomic.LoadInt64(&vs.inFlightDownloadDataSize) > vs.concurrentDownloadLimit {
|
||||
glog.V(4).Infof("wait because inflight download data %d > %d", vs.inFlightDownloadDataSize, vs.concurrentDownloadLimit)
|
||||
vs.inFlightDownloadDataLimitCond.Wait()
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
glog.V(4).Infof("request cancelled from %s: %v", r.RemoteAddr, r.Context().Err())
|
||||
return
|
||||
default:
|
||||
glog.V(4).Infof("wait because inflight download data %d > %d", vs.inFlightDownloadDataSize, vs.concurrentDownloadLimit)
|
||||
vs.inFlightDownloadDataLimitCond.Wait()
|
||||
}
|
||||
}
|
||||
vs.inFlightDownloadDataLimitCond.L.Unlock()
|
||||
vs.GetOrHeadHandler(w, r)
|
||||
@ -51,16 +58,18 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
// wait until in flight data is less than the limit
|
||||
contentLength := getContentLength(r)
|
||||
vs.inFlightUploadDataLimitCond.L.Lock()
|
||||
for vs.concurrentUploadLimit != 0 && atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit {
|
||||
glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
|
||||
vs.inFlightUploadDataLimitCond.Wait()
|
||||
|
||||
// exclude the replication from the concurrentUploadLimitMB
|
||||
if vs.concurrentUploadLimit != 0 && r.URL.Query().Get("type") != "replicate" &&
|
||||
atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit {
|
||||
err := fmt.Errorf("reject because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
|
||||
glog.V(1).Infof("too many requests: %v", err)
|
||||
writeJsonError(w, r, http.StatusTooManyRequests, err)
|
||||
return
|
||||
}
|
||||
vs.inFlightUploadDataLimitCond.L.Unlock()
|
||||
atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength)
|
||||
defer func() {
|
||||
atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength)
|
||||
vs.inFlightUploadDataLimitCond.Signal()
|
||||
}()
|
||||
|
||||
// processs uploads
|
||||
|
@ -46,7 +46,7 @@ var (
|
||||
func NewCommandEnv(options *ShellOptions) *CommandEnv {
|
||||
ce := &CommandEnv{
|
||||
env: make(map[string]string),
|
||||
MasterClient: wdclient.NewMasterClient(options.GrpcDialOption, pb.AdminShellClient, "", "", pb.ServerAddresses(*options.Masters).ToAddresses()),
|
||||
MasterClient: wdclient.NewMasterClient(options.GrpcDialOption, pb.AdminShellClient, "", "", pb.ServerAddresses(*options.Masters).ToAddressMap()),
|
||||
option: options,
|
||||
}
|
||||
ce.locker = exclusive_locks.NewExclusiveLocker(ce.MasterClient, "admin")
|
||||
|
@ -220,7 +220,7 @@ func encodeDatFile(remainingSize int64, err error, baseFileName string, bufferSi
|
||||
processedSize += largeBlockSize * DataShardsCount
|
||||
}
|
||||
for remainingSize > 0 {
|
||||
encodeData(file, enc, processedSize, smallBlockSize, buffers, outputs)
|
||||
err = encodeData(file, enc, processedSize, smallBlockSize, buffers, outputs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encode small chunk data: %v", err)
|
||||
}
|
||||
|
@ -140,9 +140,7 @@ func NewVolumeLayout(rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType
|
||||
}
|
||||
|
||||
func (vl *VolumeLayout) String() string {
|
||||
vl.accessLock.RLock()
|
||||
defer vl.accessLock.RUnlock()
|
||||
return fmt.Sprintf("rp:%v, ttl:%v, vid2location:%v, writables:%v, volumeSizeLimit:%v", vl.rp, vl.ttl, vl.vid2location, vl.writables, vl.volumeSizeLimit)
|
||||
return fmt.Sprintf("rp:%v, ttl:%v, writables:%v, volumeSizeLimit:%v", vl.rp, vl.ttl, vl.writables, vl.volumeSizeLimit)
|
||||
}
|
||||
|
||||
func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) {
|
||||
@ -220,6 +218,13 @@ func (vl *VolumeLayout) ensureCorrectWritables(vid needle.VolumeId) {
|
||||
vl.setVolumeWritable(vid)
|
||||
}
|
||||
} else {
|
||||
if !vl.enoughCopies(vid) {
|
||||
glog.V(0).Infof("volume %d does not have enough copies", vid)
|
||||
}
|
||||
if !vl.isAllWritable(vid) {
|
||||
glog.V(0).Infof("volume %d are not all writable", vid)
|
||||
}
|
||||
glog.V(0).Infof("volume %d remove from writable", vid)
|
||||
vl.removeFromWritable(vid)
|
||||
}
|
||||
}
|
||||
@ -435,7 +440,7 @@ func (vl *VolumeLayout) SetVolumeCapacityFull(vid needle.VolumeId) bool {
|
||||
vl.accessLock.Lock()
|
||||
defer vl.accessLock.Unlock()
|
||||
|
||||
// glog.V(0).Infoln("Volume", vid, "reaches full capacity.")
|
||||
glog.V(0).Infof("Volume %d reaches full capacity.", vid)
|
||||
return vl.removeFromWritable(vid)
|
||||
}
|
||||
|
||||
|
@ -9,6 +9,20 @@ import (
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
var (
|
||||
ConfigurationFileDirectory DirectoryValueType
|
||||
)
|
||||
|
||||
type DirectoryValueType string
|
||||
|
||||
func (s *DirectoryValueType) Set(value string) error {
|
||||
*s = DirectoryValueType(value)
|
||||
return nil
|
||||
}
|
||||
func (s *DirectoryValueType) String() string {
|
||||
return string(*s)
|
||||
}
|
||||
|
||||
type Configuration interface {
|
||||
GetString(key string) string
|
||||
GetBool(key string) bool
|
||||
@ -20,11 +34,12 @@ type Configuration interface {
|
||||
func LoadConfiguration(configFileName string, required bool) (loaded bool) {
|
||||
|
||||
// find a filer store
|
||||
viper.SetConfigName(configFileName) // name of config file (without extension)
|
||||
viper.AddConfigPath(".") // optionally look for config in the working directory
|
||||
viper.AddConfigPath("$HOME/.seaweedfs") // call multiple times to add many search paths
|
||||
viper.AddConfigPath("/usr/local/etc/seaweedfs/") // search path for bsd-style config directory in
|
||||
viper.AddConfigPath("/etc/seaweedfs/") // path to look for the config file in
|
||||
viper.SetConfigName(configFileName) // name of config file (without extension)
|
||||
viper.AddConfigPath(ResolvePath(ConfigurationFileDirectory.String())) // path to look for the config file in
|
||||
viper.AddConfigPath(".") // optionally look for config in the working directory
|
||||
viper.AddConfigPath("$HOME/.seaweedfs") // call multiple times to add many search paths
|
||||
viper.AddConfigPath("/usr/local/etc/seaweedfs/") // search path for bsd-style config directory in
|
||||
viper.AddConfigPath("/etc/seaweedfs/") // path to look for the config file in
|
||||
|
||||
if err := viper.MergeInConfig(); err != nil { // Handle errors reading the config file
|
||||
if strings.Contains(err.Error(), "Not Found") {
|
||||
|
@ -5,7 +5,7 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
VERSION_NUMBER = fmt.Sprintf("%.02f", 2.94)
|
||||
VERSION_NUMBER = fmt.Sprintf("%.02f", 2.96)
|
||||
VERSION = sizeLimit + " " + VERSION_NUMBER
|
||||
COMMIT = ""
|
||||
)
|
||||
|
@ -109,7 +109,7 @@ func NewIpAndLocalListeners(host string, port int, timeout time.Duration) (ipLis
|
||||
WriteTimeout: timeout,
|
||||
}
|
||||
|
||||
if host != "localhost" && host != "" {
|
||||
if host != "localhost" && host != "" && host != "0.0.0.0" && host != "127.0.0.1" {
|
||||
listner, err = net.Listen("tcp", JoinHostPort("localhost", port))
|
||||
if err != nil {
|
||||
return
|
||||
|
@ -18,7 +18,7 @@ type MasterClient struct {
|
||||
clientType string
|
||||
clientHost pb.ServerAddress
|
||||
currentMaster pb.ServerAddress
|
||||
masters []pb.ServerAddress
|
||||
masters map[string]pb.ServerAddress
|
||||
grpcDialOption grpc.DialOption
|
||||
|
||||
vidMap
|
||||
@ -26,7 +26,7 @@ type MasterClient struct {
|
||||
OnPeerUpdate func(update *master_pb.ClusterNodeUpdate)
|
||||
}
|
||||
|
||||
func NewMasterClient(grpcDialOption grpc.DialOption, clientType string, clientHost pb.ServerAddress, clientDataCenter string, masters []pb.ServerAddress) *MasterClient {
|
||||
func NewMasterClient(grpcDialOption grpc.DialOption, clientType string, clientHost pb.ServerAddress, clientDataCenter string, masters map[string]pb.ServerAddress) *MasterClient {
|
||||
return &MasterClient{
|
||||
clientType: clientType,
|
||||
clientHost: clientHost,
|
||||
|
@ -4,6 +4,7 @@ import (
|
||||
"embed"
|
||||
"fmt"
|
||||
weed_server "github.com/chrislusf/seaweedfs/weed/server"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
|
||||
"io"
|
||||
"io/fs"
|
||||
@ -40,6 +41,8 @@ var static embed.FS
|
||||
|
||||
func init() {
|
||||
weed_server.StaticFS, _ = fs.Sub(static, "static")
|
||||
|
||||
flag.Var(&util.ConfigurationFileDirectory, "config_dir", "directory with toml configuration files")
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
Loading…
Reference in New Issue
Block a user