From a73e177ecff78aee1c59b3b57a296b10a778bc9e Mon Sep 17 00:00:00 2001 From: Patrick Schmidt Date: Wed, 31 Aug 2022 18:27:53 +0200 Subject: [PATCH] Add an End-to-End workflow for FUSE mount (#3562) * Add an e2e workflow to test FUSE mount * Fix deadlocks during concurrent r/w --- .github/workflows/codeql.yml | 4 ++ .github/workflows/e2e.yml | 89 ++++++++++++++++++++++++++++++++++ docker/Dockerfile.e2e | 30 ++++++++++++ docker/Dockerfile.local | 1 + docker/Makefile | 14 ++++-- docker/compose/e2e-mount.yml | 53 ++++++++++++++++++++ weed/mount/filehandle.go | 8 +-- weed/mount/weedfs_file_read.go | 4 +- 8 files changed, 192 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/e2e.yml create mode 100644 docker/Dockerfile.e2e create mode 100644 docker/compose/e2e-mount.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 65e7c3ec6..a23a682d1 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -3,6 +3,10 @@ name: "Code Scanning - Action" on: pull_request: +concurrency: + group: ${{ github.head_ref }}/codeql + cancel-in-progress: true + jobs: CodeQL-Build: # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 000000000..6bca28820 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,89 @@ +name: "End to End" + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +concurrency: + group: ${{ github.head_ref }}/e2e + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: docker + +jobs: + e2e: + name: FUSE Mount + runs-on: ubuntu-22.04 + timeout-minutes: 15 + steps: + - name: Set up Go 1.x + uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # v2 + with: + go-version: ^1.13 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y fuse + + - name: Start SeaweedFS + timeout-minutes: 5 + run: make build_e2e && docker compose -f ./compose/e2e-mount.yml up --wait + + - name: Run FIO + timeout-minutes: 5 + run: | + echo "Starting FIO at: $(date)" + # Concurrent r/w + echo 'Run randrw with size=16M bs=4k' + docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 + + echo 'Run randrw with size=16M bs=128k' + docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 + + echo 'Run randrw with size=16M bs=1m' + docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 + + # Verified write + echo 'Run randwrite with size=16M bs=4k' + docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1 + + echo 'Run randwrite with size=16M bs=128k' + docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1 + + echo 'Run randwrite with size=16M bs=1m' + docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1 + + - name: Save logs + if: always() + run: | + docker compose -f ./compose/e2e-mount.yml logs > output.log + echo 'Showing last 500 log lines of mount service:' + docker compose -f ./compose/e2e-mount.yml logs --tail 500 mount + + - name: Check for data races + if: always() + continue-on-error: true # TODO: remove this comment to enable build failure on data races (after all are fixed) + run: grep -A50 'DATA RACE' output.log && exit 1 || exit 0 + + - name: Archive logs + if: always() + uses: actions/upload-artifact@v3 + with: + name: output-logs + path: docker/output.log + + - name: Cleanup + if: always() + run: docker compose -f ./compose/e2e-mount.yml down --volumes --remove-orphans --rmi all diff --git a/docker/Dockerfile.e2e b/docker/Dockerfile.e2e new file mode 100644 index 000000000..70f173128 --- /dev/null +++ b/docker/Dockerfile.e2e @@ -0,0 +1,30 @@ +FROM ubuntu:22.04 + +LABEL author="Chris Lu" + +RUN apt-get update && apt-get install -y curl fio fuse +RUN mkdir -p /etc/seaweedfs /data/filerldb2 + +COPY ./weed /usr/bin/ +COPY ./filer.toml /etc/seaweedfs/filer.toml +COPY ./entrypoint.sh /entrypoint.sh + +# volume server grpc port +EXPOSE 18080 +# volume server http port +EXPOSE 8080 +# filer server grpc port +EXPOSE 18888 +# filer server http port +EXPOSE 8888 +# master server shared grpc port +EXPOSE 19333 +# master server shared http port +EXPOSE 9333 + +VOLUME /data +WORKDIR /data + +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/Dockerfile.local b/docker/Dockerfile.local index 947edffda..53cfd9571 100644 --- a/docker/Dockerfile.local +++ b/docker/Dockerfile.local @@ -5,6 +5,7 @@ RUN mkdir -p /etc/seaweedfs COPY ./filer.toml /etc/seaweedfs/filer.toml COPY ./entrypoint.sh /entrypoint.sh RUN apk add fuse # for weed mount +RUN apk add curl # for health checks # volume server grpc port EXPOSE 18080 diff --git a/docker/Makefile b/docker/Makefile index 793ef17de..8c2e4d371 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -8,11 +8,17 @@ cgo ?= 0 binary: export SWCOMMIT=$(shell git rev-parse --short HEAD) export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)" - cd ../weed; CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/ + cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)" && mv weed ../docker/ + +binary_race: options = -race +binary_race: cgo = 1 +binary_race: binary build: binary docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local . - rm ./weed + +build_e2e: binary_race + docker build --no-cache -t chrislusf/seaweedfs:e2e -f Dockerfile.e2e . go_build: # make go_build tags=elastic,ydb,gocdk,hdfs,5BytesOffset docker build --build-arg TAGS=$(tags) --no-cache -t chrislusf/seaweedfs:go_build -f Dockerfile.go_build . @@ -29,9 +35,7 @@ s3tests_build: dev: build docker-compose -f compose/local-dev-compose.yml -p seaweedfs up -dev_race: options = -race -dev_race: cgo = 1 -dev_race: build +dev_race: binary_race docker-compose -f compose/local-dev-compose.yml -p seaweedfs up dev_tls: build certstrap diff --git a/docker/compose/e2e-mount.yml b/docker/compose/e2e-mount.yml new file mode 100644 index 000000000..d5da9c221 --- /dev/null +++ b/docker/compose/e2e-mount.yml @@ -0,0 +1,53 @@ +version: '3.9' + +services: + master: + image: chrislusf/seaweedfs:e2e + command: "-v=4 master -ip=master -ip.bind=0.0.0.0 -raftBootstrap" + healthcheck: + test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ] + interval: 1s + timeout: 60s + + volume: + image: chrislusf/seaweedfs:e2e + command: "-v=4 volume -mserver=master:9333 -ip=volume -ip.bind=0.0.0.0 -preStopSeconds=1" + healthcheck: + test: [ "CMD", "curl", "--fail", "-I", "http://localhost:8080/healthz" ] + interval: 1s + timeout: 30s + depends_on: + master: + condition: service_healthy + + filer: + image: chrislusf/seaweedfs:e2e + command: "-v=4 filer -master=master:9333 -ip=filer -ip.bind=0.0.0.0" + healthcheck: + test: [ "CMD", "curl", "--fail", "-I", "http://localhost:8888" ] + interval: 1s + timeout: 30s + depends_on: + volume: + condition: service_healthy + + mount: + image: chrislusf/seaweedfs:e2e + command: "-v=4 mount -filer=filer:8888 -filer.path=/ -dirAutoCreate -dir=/mnt/seaweedfs" + cap_add: + - SYS_ADMIN + devices: + - /dev/fuse + security_opt: + - apparmor:unconfined + deploy: + resources: + limits: + memory: 4096m + healthcheck: + test: [ "CMD", "mountpoint", "-q", "--", "/mnt/seaweedfs" ] + interval: 1s + timeout: 30s + depends_on: + filer: + condition: service_healthy diff --git a/weed/mount/filehandle.go b/weed/mount/filehandle.go index b2459d9e2..5d1552ce6 100644 --- a/weed/mount/filehandle.go +++ b/weed/mount/filehandle.go @@ -77,6 +77,10 @@ func (fh *FileHandle) AddChunks(chunks []*filer_pb.FileChunk) { fh.entryLock.Lock() defer fh.entryLock.Unlock() + if fh.entry == nil { + return + } + // find the earliest incoming chunk newChunks := chunks earliestChunk := newChunks[0] @@ -86,10 +90,6 @@ func (fh *FileHandle) AddChunks(chunks []*filer_pb.FileChunk) { } } - if fh.entry == nil { - return - } - // pick out-of-order chunks from existing chunks for _, chunk := range fh.entry.Chunks { if lessThan(earliestChunk, chunk) { diff --git a/weed/mount/weedfs_file_read.go b/weed/mount/weedfs_file_read.go index 0375bc206..307ad5960 100644 --- a/weed/mount/weedfs_file_read.go +++ b/weed/mount/weedfs_file_read.go @@ -39,8 +39,8 @@ func (wfs *WFS) Read(cancel <-chan struct{}, in *fuse.ReadIn, buff []byte) (fuse return nil, fuse.ENOENT } - fh.entryLock.Lock() - defer fh.entryLock.Unlock() + fh.Lock() + defer fh.Unlock() offset := int64(in.Offset) totalRead, err := readDataByFileHandle(buff, fh, offset)