mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-06-07 09:25:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into s3_baddigest

Browse Source
This commit is contained in:
Konstantin Lebedev 2025-04-23 22:46:40 +05:00 committed by GitHub
commit a81469fd3d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 96 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
go.mod
View File

@ -138,6 +138,7 @@ require (
github.com/hanwen/go-fuse/v2 v2.7.2
github.com/hashicorp/raft v1.7.3
github.com/hashicorp/raft-boltdb/v2 v2.3.1
github.com/minio/crc64nvme v1.0.1
github.com/orcaman/concurrent-map/v2 v2.0.1
github.com/parquet-go/parquet-go v0.24.0
github.com/rabbitmq/amqp091-go v1.10.0
@ -268,7 +269,7 @@ require (
github.com/josharian/intern v1.0.0 // indirect
github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/klauspost/cpuid/v2 v2.2.9 // indirect
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
github.com/kr/fs v0.1.0 // indirect

6
go.sum
View File

@ -1240,8 +1240,8 @@ github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zt
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
github.com/klauspost/reedsolomon v1.12.4 h1:5aDr3ZGoJbgu/8+j45KtUJxzYm8k08JGtB9Wx1VQ4OA=
github.com/klauspost/reedsolomon v1.12.4/go.mod h1:d3CzOMOt0JXGIFZm1StgkyF14EYr3xneR2rNWo7NcMU=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@ -1301,6 +1301,8 @@ github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY=
github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=

4
weed/s3api/chunked_reader_v4.go
View File

@ -29,7 +29,6 @@ import (
"fmt"
"hash"
"hash/crc32"
"hash/crc64"
"io"
"net/http"
"time"
@ -39,6 +38,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/dustin/go-humanize"
"github.com/minio/crc64nvme"
)
// calculateSeedSignature - Calculate seed signature in accordance with
@ -673,7 +673,7 @@ func getCheckSumWriter(checksumAlgorithm ChecksumAlgorithm) hash.Hash {
case ChecksumAlgorithmCRC32C:
return crc32.New(crc32.MakeTable(crc32.Castagnoli))
case ChecksumAlgorithmCRC64NVMe:
return crc64.New(crc64.MakeTable(crc64.ISO))
return crc64nvme.New()
case ChecksumAlgorithmSHA1:
return sha1.New()
case ChecksumAlgorithmSHA256:

18
weed/topology/topology.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"math/rand/v2"
"slices"
"sync"
"time"
@ -268,23 +269,29 @@ func (t *Topology) GetVolumeLayout(collectionName string, rp *super_block.Replic
}
func (t *Topology) ListCollections(includeNormalVolumes, includeEcVolumes bool) (ret []string) {
found := make(map[string]bool)
mapOfCollections := make(map[string]bool)
for _, c := range t.collectionMap.Items() {
mapOfCollections[c.(*Collection).Name] = true
if includeNormalVolumes {
t.collectionMap.RLock()
for _, c := range t.collectionMap.Items() {
found[c.(*Collection).Name] = true
}
t.collectionMap.RUnlock()
}
if includeEcVolumes {
t.ecShardMapLock.RLock()
for _, ecVolumeLocation := range t.ecShardMap {
mapOfCollections[ecVolumeLocation.Collection] = true
found[ecVolumeLocation.Collection] = true
}
t.ecShardMapLock.RUnlock()
}
for k := range mapOfCollections {
for k := range found {
ret = append(ret, k)
}
slices.Sort(ret)
return ret
}
@ -317,6 +324,7 @@ func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) {
vl.RegisterVolume(&v, dn)
vl.EnsureCorrectWritables(&v)
}
func (t *Topology) UnRegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) {
glog.Infof("removing volume info: %+v from %v", v, dn.id)
if v.ReplicaPlacement.GetCopyCount() > 1 {

76
weed/topology/topology_test.go
View File

@ -1,9 +1,12 @@
package topology
import (
"reflect"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/sequence"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
@ -206,5 +209,76 @@ func TestAddRemoveVolume(t *testing.T) {
if _, hasCollection := topo.FindCollection(v.Collection); hasCollection {
t.Errorf("collection %v should not exist", v.Collection)
}
}
func TestListCollections(t *testing.T) {
rp, _ := super_block.NewReplicaPlacementFromString("002")
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc1")
rack := dc.GetOrCreateRack("rack1")
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", nil)
topo.RegisterVolumeLayout(storage.VolumeInfo{
Id: needle.VolumeId(1111),
ReplicaPlacement: rp,
}, dn)
topo.RegisterVolumeLayout(storage.VolumeInfo{
Id: needle.VolumeId(2222),
ReplicaPlacement: rp,
Collection: "vol_collection_a",
}, dn)
topo.RegisterVolumeLayout(storage.VolumeInfo{
Id: needle.VolumeId(3333),
ReplicaPlacement: rp,
Collection: "vol_collection_b",
}, dn)
topo.RegisterEcShards(&erasure_coding.EcVolumeInfo{
VolumeId: needle.VolumeId(4444),
Collection: "ec_collection_a",
}, dn)
topo.RegisterEcShards(&erasure_coding.EcVolumeInfo{
VolumeId: needle.VolumeId(5555),
Collection: "ec_collection_b",
}, dn)
testCases := []struct {
name string
includeNormalVolumes bool
includeEcVolumes bool
want []string
}{
{
name: "no volume types selected",
includeNormalVolumes: false,
includeEcVolumes: false,
want: nil,
}, {
name: "normal volumes",
includeNormalVolumes: true,
includeEcVolumes: false,
want: []string{"", "vol_collection_a", "vol_collection_b"},
}, {
name: "EC volumes",
includeNormalVolumes: false,
includeEcVolumes: true,
want: []string{"ec_collection_a", "ec_collection_b"},
}, {
name: "normal + EC volumes",
includeNormalVolumes: true,
includeEcVolumes: true,
want: []string{"", "ec_collection_a", "ec_collection_b", "vol_collection_a", "vol_collection_b"},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
got := topo.ListCollections(tc.includeNormalVolumes, tc.includeEcVolumes)
if !reflect.DeepEqual(got, tc.want) {
t.Errorf("got %v, want %v", got, tc.want)
}
})
}
}