2011-12-16 22:51:26 +08:00
|
|
|
package storage
|
|
|
|
|
|
|
|
import (
|
2013-07-04 13:14:16 +08:00
|
|
|
"fmt"
|
2022-06-06 02:54:04 +08:00
|
|
|
"io"
|
2020-02-03 07:37:23 +08:00
|
|
|
"path/filepath"
|
|
|
|
"strings"
|
2022-10-08 02:27:15 +08:00
|
|
|
"sync"
|
2019-04-15 14:00:37 +08:00
|
|
|
"sync/atomic"
|
|
|
|
|
2022-07-29 15:17:28 +08:00
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
2022-05-16 10:41:18 +08:00
|
|
|
|
2019-12-19 16:42:46 +08:00
|
|
|
"google.golang.org/grpc"
|
|
|
|
|
2022-07-29 15:17:28 +08:00
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
|
|
|
. "github.com/seaweedfs/seaweedfs/weed/storage/types"
|
2011-12-16 22:51:26 +08:00
|
|
|
)
|
|
|
|
|
2014-09-21 03:38:59 +08:00
|
|
|
const (
|
|
|
|
MAX_TTL_VOLUME_REMOVAL_DELAY = 10 // 10 minutes
|
|
|
|
)
|
|
|
|
|
2020-08-19 08:37:26 +08:00
|
|
|
type ReadOption struct {
|
2022-06-06 06:24:02 +08:00
|
|
|
// request
|
2022-06-06 02:54:04 +08:00
|
|
|
ReadDeleted bool
|
|
|
|
AttemptMetaOnly bool
|
|
|
|
MustMetaOnly bool
|
2022-06-06 06:24:02 +08:00
|
|
|
// response
|
|
|
|
IsMetaOnly bool // read status
|
|
|
|
VolumeRevision uint16
|
|
|
|
IsOutOfRange bool // whether read over MaxPossibleVolumeSize
|
2022-09-15 18:11:32 +08:00
|
|
|
|
|
|
|
// If HasSlowRead is set to true:
|
|
|
|
// * read requests and write requests compete for the lock.
|
|
|
|
// * large file read P99 latency on busy sites will go up, due to the need to get locks multiple times.
|
|
|
|
// * write requests will see lower latency.
|
|
|
|
// If HasSlowRead is set to false:
|
|
|
|
// * read requests should complete asap, not blocking other requests.
|
|
|
|
// * write requests may see high latency when downloading large files.
|
|
|
|
HasSlowRead bool
|
2022-09-16 15:30:40 +08:00
|
|
|
|
|
|
|
// increasing ReadBufferSize can reduce the number of get locks times and shorten read P99 latency.
|
|
|
|
// but will increase memory usage a bit. Use with hasSlowRead normally.
|
|
|
|
ReadBufferSize int
|
2020-08-19 08:37:26 +08:00
|
|
|
}
|
|
|
|
|
2014-12-26 15:36:33 +08:00
|
|
|
/*
|
|
|
|
* A VolumeServer contains one Store
|
|
|
|
*/
|
2013-07-14 02:38:01 +08:00
|
|
|
type Store struct {
|
2021-09-13 13:47:52 +08:00
|
|
|
MasterAddress pb.ServerAddress
|
2019-05-28 12:22:23 +08:00
|
|
|
grpcDialOption grpc.DialOption
|
2020-07-17 13:50:14 +08:00
|
|
|
volumeSizeLimit uint64 // read from the master
|
2019-05-27 16:29:46 +08:00
|
|
|
Ip string
|
|
|
|
Port int
|
2021-09-13 13:47:52 +08:00
|
|
|
GrpcPort int
|
2019-05-27 16:29:46 +08:00
|
|
|
PublicUrl string
|
|
|
|
Locations []*DiskLocation
|
2022-09-16 17:43:17 +08:00
|
|
|
dataCenter string // optional information, overwriting master setting if exists
|
2020-07-17 13:50:14 +08:00
|
|
|
rack string // optional information, overwriting master setting if exists
|
2019-05-27 16:29:46 +08:00
|
|
|
connected bool
|
2021-02-07 09:00:03 +08:00
|
|
|
NeedleMapKind NeedleMapKind
|
2019-05-27 16:29:46 +08:00
|
|
|
NewVolumesChan chan master_pb.VolumeShortInformationMessage
|
|
|
|
DeletedVolumesChan chan master_pb.VolumeShortInformationMessage
|
2019-05-22 13:41:20 +08:00
|
|
|
NewEcShardsChan chan master_pb.VolumeEcShardInformationMessage
|
|
|
|
DeletedEcShardsChan chan master_pb.VolumeEcShardInformationMessage
|
2021-07-13 16:29:57 +08:00
|
|
|
isStopping bool
|
2011-12-16 22:51:26 +08:00
|
|
|
}
|
|
|
|
|
2014-12-26 15:36:33 +08:00
|
|
|
func (s *Store) String() (str string) {
|
2021-09-13 13:47:52 +08:00
|
|
|
str = fmt.Sprintf("Ip:%s, Port:%d, GrpcPort:%d PublicUrl:%s, dataCenter:%s, rack:%s, connected:%v, volumeSizeLimit:%d", s.Ip, s.Port, s.GrpcPort, s.PublicUrl, s.dataCenter, s.rack, s.connected, s.GetVolumeSizeLimit())
|
2014-12-26 15:36:33 +08:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2022-08-27 08:09:11 +08:00
|
|
|
func NewStore(grpcDialOption grpc.DialOption, ip string, port int, grpcPort int, publicUrl string, dirnames []string, maxVolumeCounts []int32,
|
2022-11-14 16:19:27 +08:00
|
|
|
minFreeSpaces []util.MinFreeSpace, idxFolder string, needleMapKind NeedleMapKind, diskTypes []DiskType, ldbTimeout int64) (s *Store) {
|
2021-09-13 13:47:52 +08:00
|
|
|
s = &Store{grpcDialOption: grpcDialOption, Port: port, Ip: ip, GrpcPort: grpcPort, PublicUrl: publicUrl, NeedleMapKind: needleMapKind}
|
2014-03-27 04:22:27 +08:00
|
|
|
s.Locations = make([]*DiskLocation, 0)
|
2022-10-08 02:27:15 +08:00
|
|
|
|
|
|
|
var wg sync.WaitGroup
|
2013-07-14 02:38:01 +08:00
|
|
|
for i := 0; i < len(dirnames); i++ {
|
2022-08-26 23:41:42 +08:00
|
|
|
location := NewDiskLocation(dirnames[i], int32(maxVolumeCounts[i]), minFreeSpaces[i], idxFolder, diskTypes[i])
|
2014-03-27 04:22:27 +08:00
|
|
|
s.Locations = append(s.Locations, location)
|
2019-06-18 12:02:50 +08:00
|
|
|
stats.VolumeServerMaxVolumeCounter.Add(float64(maxVolumeCounts[i]))
|
2022-10-08 02:27:15 +08:00
|
|
|
|
|
|
|
wg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer wg.Done()
|
2022-11-14 16:19:27 +08:00
|
|
|
location.loadExistingVolumes(needleMapKind, ldbTimeout)
|
2022-10-08 02:27:15 +08:00
|
|
|
}()
|
2013-07-14 02:38:01 +08:00
|
|
|
}
|
2022-10-08 02:27:15 +08:00
|
|
|
wg.Wait()
|
|
|
|
|
2019-04-21 02:35:20 +08:00
|
|
|
s.NewVolumesChan = make(chan master_pb.VolumeShortInformationMessage, 3)
|
|
|
|
s.DeletedVolumesChan = make(chan master_pb.VolumeShortInformationMessage, 3)
|
2019-05-22 13:41:20 +08:00
|
|
|
|
|
|
|
s.NewEcShardsChan = make(chan master_pb.VolumeEcShardInformationMessage, 3)
|
|
|
|
s.DeletedEcShardsChan = make(chan master_pb.VolumeEcShardInformationMessage, 3)
|
|
|
|
|
2011-12-29 09:46:38 +08:00
|
|
|
return
|
|
|
|
}
|
2022-11-14 16:19:27 +08:00
|
|
|
func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapKind, replicaPlacement string, ttlString string, preallocate int64, MemoryMapMaxSizeMb uint32, diskType DiskType, ldbTimeout int64) error {
|
2019-12-24 04:48:20 +08:00
|
|
|
rt, e := super_block.NewReplicaPlacementFromString(replicaPlacement)
|
2012-09-26 16:55:56 +08:00
|
|
|
if e != nil {
|
|
|
|
return e
|
|
|
|
}
|
2019-04-19 12:43:36 +08:00
|
|
|
ttl, e := needle.ReadTTL(ttlString)
|
2014-09-21 03:38:59 +08:00
|
|
|
if e != nil {
|
|
|
|
return e
|
|
|
|
}
|
2022-11-14 16:19:27 +08:00
|
|
|
e = s.addVolume(volumeId, collection, needleMapKind, rt, ttl, preallocate, MemoryMapMaxSizeMb, diskType, ldbTimeout)
|
2012-09-13 15:04:56 +08:00
|
|
|
return e
|
2011-12-29 09:46:38 +08:00
|
|
|
}
|
2014-03-11 02:43:54 +08:00
|
|
|
func (s *Store) DeleteCollection(collection string) (e error) {
|
2014-03-27 04:22:27 +08:00
|
|
|
for _, location := range s.Locations {
|
2016-04-27 11:45:35 +08:00
|
|
|
e = location.DeleteCollectionFromDiskLocation(collection)
|
|
|
|
if e != nil {
|
|
|
|
return
|
2014-03-11 02:43:54 +08:00
|
|
|
}
|
2022-07-27 19:31:49 +08:00
|
|
|
stats.DeleteCollectionMetrics(collection)
|
2019-04-21 02:35:20 +08:00
|
|
|
// let the heartbeat send the list of volumes, instead of sending the deleted volume ids to DeletedVolumesChan
|
2014-03-11 02:43:54 +08:00
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
2016-04-27 11:45:35 +08:00
|
|
|
|
2019-04-19 12:43:36 +08:00
|
|
|
func (s *Store) findVolume(vid needle.VolumeId) *Volume {
|
2014-03-27 04:22:27 +08:00
|
|
|
for _, location := range s.Locations {
|
2016-11-07 12:55:22 +08:00
|
|
|
if v, found := location.FindVolume(vid); found {
|
2013-07-14 02:38:01 +08:00
|
|
|
return v
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
2020-12-14 15:08:21 +08:00
|
|
|
func (s *Store) FindFreeLocation(diskType DiskType) (ret *DiskLocation) {
|
2022-08-26 23:41:42 +08:00
|
|
|
max := int32(0)
|
2014-03-27 04:22:27 +08:00
|
|
|
for _, location := range s.Locations {
|
2020-12-14 15:39:00 +08:00
|
|
|
if diskType != location.DiskType {
|
|
|
|
continue
|
|
|
|
}
|
2021-04-22 14:11:09 +08:00
|
|
|
if location.isDiskSpaceLow {
|
|
|
|
continue
|
|
|
|
}
|
2022-08-26 23:41:42 +08:00
|
|
|
currentFreeCount := location.MaxVolumeCount - int32(location.VolumesLen())
|
2020-03-23 07:21:42 +08:00
|
|
|
currentFreeCount *= erasure_coding.DataShardsCount
|
2024-01-12 00:19:09 +08:00
|
|
|
currentFreeCount -= int32(location.EcShardCount())
|
2020-03-23 07:21:42 +08:00
|
|
|
currentFreeCount /= erasure_coding.DataShardsCount
|
2013-07-14 02:38:01 +08:00
|
|
|
if currentFreeCount > max {
|
|
|
|
max = currentFreeCount
|
|
|
|
ret = location
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ret
|
|
|
|
}
|
2022-11-14 16:19:27 +08:00
|
|
|
func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind NeedleMapKind, replicaPlacement *super_block.ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapMaxSizeMb uint32, diskType DiskType, ldbTimeout int64) error {
|
2013-07-14 02:38:01 +08:00
|
|
|
if s.findVolume(vid) != nil {
|
2014-04-17 15:16:44 +08:00
|
|
|
return fmt.Errorf("Volume Id %d already exists!", vid)
|
2012-06-29 15:53:47 +08:00
|
|
|
}
|
2020-12-14 15:08:21 +08:00
|
|
|
if location := s.FindFreeLocation(diskType); location != nil {
|
2014-09-21 03:38:59 +08:00
|
|
|
glog.V(0).Infof("In dir %s adds volume:%v collection:%s replicaPlacement:%v ttl:%v",
|
|
|
|
location.Directory, vid, collection, replicaPlacement, ttl)
|
2022-11-14 16:19:27 +08:00
|
|
|
if volume, err := NewVolume(location.Directory, location.IdxDirectory, collection, vid, needleMapKind, replicaPlacement, ttl, preallocate, memoryMapMaxSizeMb, ldbTimeout); err == nil {
|
2016-11-07 12:55:22 +08:00
|
|
|
location.SetVolume(vid, volume)
|
2019-04-22 01:14:17 +08:00
|
|
|
glog.V(0).Infof("add volume %d", vid)
|
2019-04-21 02:35:20 +08:00
|
|
|
s.NewVolumesChan <- master_pb.VolumeShortInformationMessage{
|
|
|
|
Id: uint32(vid),
|
|
|
|
Collection: collection,
|
|
|
|
ReplicaPlacement: uint32(replicaPlacement.Byte()),
|
2019-04-21 14:53:37 +08:00
|
|
|
Version: uint32(volume.Version()),
|
2019-04-21 02:35:20 +08:00
|
|
|
Ttl: ttl.ToUint32(),
|
2020-12-17 01:10:14 +08:00
|
|
|
DiskType: string(diskType),
|
2019-04-21 02:35:20 +08:00
|
|
|
}
|
2013-09-02 14:58:21 +08:00
|
|
|
return nil
|
2013-07-20 11:38:00 +08:00
|
|
|
} else {
|
2013-09-02 14:58:21 +08:00
|
|
|
return err
|
2013-07-20 11:38:00 +08:00
|
|
|
}
|
2013-07-14 02:38:01 +08:00
|
|
|
}
|
|
|
|
return fmt.Errorf("No more free space left")
|
2011-12-16 22:51:26 +08:00
|
|
|
}
|
2012-11-07 17:51:43 +08:00
|
|
|
|
2020-03-25 09:41:25 +08:00
|
|
|
func (s *Store) VolumeInfos() (allStats []*VolumeInfo) {
|
2014-03-27 04:22:27 +08:00
|
|
|
for _, location := range s.Locations {
|
2020-03-25 09:41:25 +08:00
|
|
|
stats := collectStatsForOneLocation(location)
|
|
|
|
allStats = append(allStats, stats...)
|
|
|
|
}
|
|
|
|
sortVolumeInfos(allStats)
|
|
|
|
return allStats
|
|
|
|
}
|
|
|
|
|
|
|
|
func collectStatsForOneLocation(location *DiskLocation) (stats []*VolumeInfo) {
|
|
|
|
location.volumesLock.RLock()
|
|
|
|
defer location.volumesLock.RUnlock()
|
|
|
|
|
|
|
|
for k, v := range location.volumes {
|
|
|
|
s := collectStatForOneVolume(k, v)
|
|
|
|
stats = append(stats, s)
|
2011-12-26 17:43:17 +08:00
|
|
|
}
|
|
|
|
return stats
|
2011-12-22 12:04:47 +08:00
|
|
|
}
|
2012-12-04 14:54:08 +08:00
|
|
|
|
2020-03-25 09:41:25 +08:00
|
|
|
func collectStatForOneVolume(vid needle.VolumeId, v *Volume) (s *VolumeInfo) {
|
|
|
|
|
|
|
|
s = &VolumeInfo{
|
|
|
|
Id: vid,
|
|
|
|
Collection: v.Collection,
|
|
|
|
ReplicaPlacement: v.ReplicaPlacement,
|
|
|
|
Version: v.Version(),
|
|
|
|
ReadOnly: v.IsReadOnly(),
|
|
|
|
Ttl: v.Ttl,
|
|
|
|
CompactRevision: uint32(v.CompactionRevision),
|
2021-02-14 07:42:42 +08:00
|
|
|
DiskType: v.DiskType().String(),
|
2020-03-25 09:41:25 +08:00
|
|
|
}
|
|
|
|
s.RemoteStorageName, s.RemoteStorageKey = v.RemoteStorageNameKey()
|
|
|
|
|
|
|
|
v.dataFileAccessLock.RLock()
|
|
|
|
defer v.dataFileAccessLock.RUnlock()
|
|
|
|
|
|
|
|
if v.nm == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
s.FileCount = v.nm.FileCount()
|
|
|
|
s.DeleteCount = v.nm.DeletedCount()
|
|
|
|
s.DeletedByteCount = v.nm.DeletedSize()
|
|
|
|
s.Size = v.nm.ContentSize()
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2013-06-20 09:10:38 +08:00
|
|
|
func (s *Store) SetDataCenter(dataCenter string) {
|
|
|
|
s.dataCenter = dataCenter
|
|
|
|
}
|
|
|
|
func (s *Store) SetRack(rack string) {
|
|
|
|
s.rack = rack
|
|
|
|
}
|
2020-10-31 05:48:25 +08:00
|
|
|
func (s *Store) GetDataCenter() string {
|
|
|
|
return s.dataCenter
|
|
|
|
}
|
|
|
|
func (s *Store) GetRack() string {
|
|
|
|
return s.rack
|
|
|
|
}
|
2014-02-15 09:10:49 +08:00
|
|
|
|
2018-05-10 14:11:54 +08:00
|
|
|
func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
|
|
|
|
var volumeMessages []*master_pb.VolumeInformationMessage
|
2021-02-16 18:47:02 +08:00
|
|
|
maxVolumeCounts := make(map[string]uint32)
|
2018-07-08 17:28:04 +08:00
|
|
|
var maxFileKey NeedleId
|
2022-08-18 17:39:07 +08:00
|
|
|
collectionVolumeSize := make(map[string]int64)
|
2024-03-06 23:28:46 +08:00
|
|
|
collectionVolumeDeletedBytes := make(map[string]int64)
|
2020-10-28 20:38:26 +08:00
|
|
|
collectionVolumeReadOnlyCount := make(map[string]map[string]uint8)
|
2014-03-27 04:22:27 +08:00
|
|
|
for _, location := range s.Locations {
|
2019-11-08 19:00:47 +08:00
|
|
|
var deleteVids []needle.VolumeId
|
2021-02-16 18:47:02 +08:00
|
|
|
maxVolumeCounts[string(location.DiskType)] += uint32(location.MaxVolumeCount)
|
2019-12-03 12:49:50 +08:00
|
|
|
location.volumesLock.RLock()
|
2019-03-18 11:27:08 +08:00
|
|
|
for _, v := range location.volumes {
|
2020-10-25 10:40:35 +08:00
|
|
|
curMaxFileKey, volumeMessage := v.ToVolumeInformationMessage()
|
2021-03-14 03:05:29 +08:00
|
|
|
if volumeMessage == nil {
|
|
|
|
continue
|
|
|
|
}
|
2020-10-25 10:40:35 +08:00
|
|
|
if maxFileKey < curMaxFileKey {
|
|
|
|
maxFileKey = curMaxFileKey
|
2014-04-17 14:43:27 +08:00
|
|
|
}
|
2022-07-27 16:55:09 +08:00
|
|
|
shouldDeleteVolume := false
|
2020-10-25 10:40:35 +08:00
|
|
|
if !v.expired(volumeMessage.Size, s.GetVolumeSizeLimit()) {
|
|
|
|
volumeMessages = append(volumeMessages, volumeMessage)
|
2014-09-21 03:38:59 +08:00
|
|
|
} else {
|
2019-01-17 09:17:19 +08:00
|
|
|
if v.expiredLongEnough(MAX_TTL_VOLUME_REMOVAL_DELAY) {
|
2019-11-08 19:00:47 +08:00
|
|
|
deleteVids = append(deleteVids, v.Id)
|
2022-07-27 16:55:09 +08:00
|
|
|
shouldDeleteVolume = true
|
2014-09-21 03:38:59 +08:00
|
|
|
} else {
|
2020-11-28 19:03:43 +08:00
|
|
|
glog.V(0).Infof("volume %d is expired", v.Id)
|
2020-11-28 16:09:29 +08:00
|
|
|
}
|
|
|
|
if v.lastIoError != nil {
|
|
|
|
deleteVids = append(deleteVids, v.Id)
|
2022-07-27 16:55:09 +08:00
|
|
|
shouldDeleteVolume = true
|
2020-12-12 08:57:53 +08:00
|
|
|
glog.Warningf("volume %d has IO error: %v", v.Id, v.lastIoError)
|
2014-09-21 03:38:59 +08:00
|
|
|
}
|
|
|
|
}
|
2021-03-09 18:49:45 +08:00
|
|
|
|
|
|
|
if _, exist := collectionVolumeSize[v.Collection]; !exist {
|
|
|
|
collectionVolumeSize[v.Collection] = 0
|
2024-03-06 23:28:46 +08:00
|
|
|
collectionVolumeDeletedBytes[v.Collection] = 0
|
2021-03-09 18:49:45 +08:00
|
|
|
}
|
2022-07-27 16:55:09 +08:00
|
|
|
if !shouldDeleteVolume {
|
2022-08-18 17:39:07 +08:00
|
|
|
collectionVolumeSize[v.Collection] += int64(volumeMessage.Size)
|
2024-03-06 23:28:46 +08:00
|
|
|
collectionVolumeDeletedBytes[v.Collection] += int64(volumeMessage.DeletedByteCount)
|
2021-06-16 12:11:31 +08:00
|
|
|
} else {
|
2022-08-18 17:39:07 +08:00
|
|
|
collectionVolumeSize[v.Collection] -= int64(volumeMessage.Size)
|
|
|
|
if collectionVolumeSize[v.Collection] <= 0 {
|
|
|
|
delete(collectionVolumeSize, v.Collection)
|
2021-06-16 12:11:31 +08:00
|
|
|
}
|
2021-03-09 18:49:45 +08:00
|
|
|
}
|
|
|
|
|
2020-10-28 20:38:26 +08:00
|
|
|
if _, exist := collectionVolumeReadOnlyCount[v.Collection]; !exist {
|
|
|
|
collectionVolumeReadOnlyCount[v.Collection] = map[string]uint8{
|
2022-07-27 19:31:49 +08:00
|
|
|
stats.IsReadOnly: 0,
|
|
|
|
stats.NoWriteOrDelete: 0,
|
|
|
|
stats.NoWriteCanDelete: 0,
|
|
|
|
stats.IsDiskSpaceLow: 0,
|
2020-10-28 20:38:26 +08:00
|
|
|
}
|
|
|
|
}
|
2022-07-27 16:55:09 +08:00
|
|
|
if !shouldDeleteVolume && v.IsReadOnly() {
|
2022-07-27 19:31:49 +08:00
|
|
|
collectionVolumeReadOnlyCount[v.Collection][stats.IsReadOnly] += 1
|
2020-10-28 20:38:26 +08:00
|
|
|
if v.noWriteOrDelete {
|
2022-07-27 19:31:49 +08:00
|
|
|
collectionVolumeReadOnlyCount[v.Collection][stats.NoWriteOrDelete] += 1
|
2020-10-28 20:38:26 +08:00
|
|
|
}
|
|
|
|
if v.noWriteCanDelete {
|
2022-07-27 19:31:49 +08:00
|
|
|
collectionVolumeReadOnlyCount[v.Collection][stats.NoWriteCanDelete] += 1
|
2020-10-28 20:38:26 +08:00
|
|
|
}
|
|
|
|
if v.location.isDiskSpaceLow {
|
2022-07-27 19:31:49 +08:00
|
|
|
collectionVolumeReadOnlyCount[v.Collection][stats.IsDiskSpaceLow] += 1
|
2020-10-25 19:21:40 +08:00
|
|
|
}
|
2020-10-15 18:32:02 +08:00
|
|
|
}
|
2013-07-14 02:38:01 +08:00
|
|
|
}
|
2019-12-03 12:49:50 +08:00
|
|
|
location.volumesLock.RUnlock()
|
2019-11-08 19:00:47 +08:00
|
|
|
|
|
|
|
if len(deleteVids) > 0 {
|
2019-11-09 14:47:50 +08:00
|
|
|
// delete expired volumes.
|
2019-12-03 12:49:50 +08:00
|
|
|
location.volumesLock.Lock()
|
2019-11-08 19:00:47 +08:00
|
|
|
for _, vid := range deleteVids {
|
2023-06-15 05:39:58 +08:00
|
|
|
found, err := location.deleteVolumeById(vid, false)
|
2020-12-18 11:37:46 +08:00
|
|
|
if err == nil {
|
|
|
|
if found {
|
2020-04-22 05:49:58 +08:00
|
|
|
glog.V(0).Infof("volume %d is deleted", vid)
|
|
|
|
}
|
2020-12-18 11:37:46 +08:00
|
|
|
} else {
|
2021-03-09 18:49:45 +08:00
|
|
|
glog.Warningf("delete volume %d: %v", vid, err)
|
2020-04-22 05:49:58 +08:00
|
|
|
}
|
2019-11-08 19:00:47 +08:00
|
|
|
}
|
2019-12-03 12:49:50 +08:00
|
|
|
location.volumesLock.Unlock()
|
2019-11-08 19:00:47 +08:00
|
|
|
}
|
2011-12-16 22:51:26 +08:00
|
|
|
}
|
2019-06-17 12:56:41 +08:00
|
|
|
|
2022-05-17 14:51:01 +08:00
|
|
|
var uuidList []string
|
2022-05-17 10:19:35 +08:00
|
|
|
for _, loc := range s.Locations {
|
2022-05-17 14:51:01 +08:00
|
|
|
uuidList = append(uuidList, loc.DirectoryUuid)
|
2022-05-16 10:41:18 +08:00
|
|
|
}
|
|
|
|
|
2019-06-17 12:56:41 +08:00
|
|
|
for col, size := range collectionVolumeSize {
|
|
|
|
stats.VolumeServerDiskSizeGauge.WithLabelValues(col, "normal").Set(float64(size))
|
|
|
|
}
|
2014-04-21 17:11:10 +08:00
|
|
|
|
2024-03-06 23:28:46 +08:00
|
|
|
for col, deletedBytes := range collectionVolumeDeletedBytes{
|
|
|
|
stats.VolumeServerDiskSizeGauge.WithLabelValues(col, "deleted_bytes").Set(float64(deletedBytes))
|
|
|
|
}
|
|
|
|
|
2020-10-28 20:38:26 +08:00
|
|
|
for col, types := range collectionVolumeReadOnlyCount {
|
|
|
|
for t, count := range types {
|
|
|
|
stats.VolumeServerReadOnlyVolumeGauge.WithLabelValues(col, t).Set(float64(count))
|
|
|
|
}
|
2020-10-15 18:32:02 +08:00
|
|
|
}
|
|
|
|
|
2018-05-10 14:11:54 +08:00
|
|
|
return &master_pb.Heartbeat{
|
2021-02-18 12:57:08 +08:00
|
|
|
Ip: s.Ip,
|
|
|
|
Port: uint32(s.Port),
|
2021-09-13 13:47:52 +08:00
|
|
|
GrpcPort: uint32(s.GrpcPort),
|
2021-02-18 12:57:08 +08:00
|
|
|
PublicUrl: s.PublicUrl,
|
|
|
|
MaxVolumeCounts: maxVolumeCounts,
|
|
|
|
MaxFileKey: NeedleIdToUint64(maxFileKey),
|
|
|
|
DataCenter: s.dataCenter,
|
|
|
|
Rack: s.rack,
|
|
|
|
Volumes: volumeMessages,
|
|
|
|
HasNoVolumes: len(volumeMessages) == 0,
|
2022-05-17 14:51:01 +08:00
|
|
|
LocationUuids: uuidList,
|
2012-12-23 08:26:02 +08:00
|
|
|
}
|
2014-04-21 17:11:10 +08:00
|
|
|
|
2011-12-16 22:51:26 +08:00
|
|
|
}
|
2019-05-22 13:41:20 +08:00
|
|
|
|
2021-07-13 16:29:57 +08:00
|
|
|
func (s *Store) SetStopping() {
|
|
|
|
s.isStopping = true
|
2022-02-17 01:11:34 +08:00
|
|
|
for _, location := range s.Locations {
|
|
|
|
location.SetStopping()
|
|
|
|
}
|
2021-07-13 16:29:57 +08:00
|
|
|
}
|
|
|
|
|
2022-09-29 03:44:13 +08:00
|
|
|
func (s *Store) LoadNewVolumes() {
|
|
|
|
for _, location := range s.Locations {
|
2022-11-14 16:19:27 +08:00
|
|
|
location.loadExistingVolumes(s.NeedleMapKind, 0)
|
2022-09-29 03:44:13 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-12-16 22:51:26 +08:00
|
|
|
func (s *Store) Close() {
|
2014-03-27 04:22:27 +08:00
|
|
|
for _, location := range s.Locations {
|
2016-11-07 12:55:22 +08:00
|
|
|
location.Close()
|
2011-12-16 22:51:26 +08:00
|
|
|
}
|
|
|
|
}
|
2017-01-20 19:49:20 +08:00
|
|
|
|
2021-08-13 17:09:35 +08:00
|
|
|
func (s *Store) WriteVolumeNeedle(i needle.VolumeId, n *needle.Needle, checkCookie bool, fsync bool) (isUnchanged bool, err error) {
|
2013-07-14 02:38:01 +08:00
|
|
|
if v := s.findVolume(i); v != nil {
|
2020-03-18 01:01:24 +08:00
|
|
|
if v.IsReadOnly() {
|
2019-05-26 16:14:42 +08:00
|
|
|
err = fmt.Errorf("volume %d is read only", i)
|
2013-04-18 15:23:14 +08:00
|
|
|
return
|
2015-03-10 15:20:31 +08:00
|
|
|
}
|
2021-08-13 17:09:35 +08:00
|
|
|
_, _, isUnchanged, err = v.writeNeedle2(n, checkCookie, fsync && s.isStopping)
|
2013-01-20 19:40:04 +08:00
|
|
|
return
|
2012-09-11 08:08:52 +08:00
|
|
|
}
|
2013-08-09 14:57:22 +08:00
|
|
|
glog.V(0).Infoln("volume", i, "not found!")
|
2019-04-22 01:14:17 +08:00
|
|
|
err = fmt.Errorf("volume %d not found on %s:%d", i, s.Ip, s.Port)
|
2013-01-20 19:40:04 +08:00
|
|
|
return
|
2012-01-19 08:49:41 +08:00
|
|
|
}
|
2017-01-20 19:49:20 +08:00
|
|
|
|
2020-08-19 08:04:28 +08:00
|
|
|
func (s *Store) DeleteVolumeNeedle(i needle.VolumeId, n *needle.Needle) (Size, error) {
|
2019-07-22 02:21:30 +08:00
|
|
|
if v := s.findVolume(i); v != nil {
|
2019-12-19 16:42:46 +08:00
|
|
|
if v.noWriteOrDelete {
|
2019-07-22 02:21:30 +08:00
|
|
|
return 0, fmt.Errorf("volume %d is read only", i)
|
|
|
|
}
|
2020-05-07 06:37:17 +08:00
|
|
|
return v.deleteNeedle2(n)
|
2012-09-11 08:08:52 +08:00
|
|
|
}
|
2019-07-22 02:21:30 +08:00
|
|
|
return 0, fmt.Errorf("volume %d not found on %s:%d", i, s.Ip, s.Port)
|
2011-12-16 22:51:26 +08:00
|
|
|
}
|
2017-01-20 19:49:20 +08:00
|
|
|
|
2021-08-09 14:25:16 +08:00
|
|
|
func (s *Store) ReadVolumeNeedle(i needle.VolumeId, n *needle.Needle, readOption *ReadOption, onReadSizeFn func(size Size)) (int, error) {
|
2013-07-14 02:38:01 +08:00
|
|
|
if v := s.findVolume(i); v != nil {
|
2021-08-09 14:25:16 +08:00
|
|
|
return v.readNeedle(n, readOption, onReadSizeFn)
|
2012-09-11 08:08:52 +08:00
|
|
|
}
|
2019-05-26 16:14:42 +08:00
|
|
|
return 0, fmt.Errorf("volume %d not found", i)
|
2012-09-11 08:08:52 +08:00
|
|
|
}
|
2022-09-07 14:51:27 +08:00
|
|
|
|
|
|
|
func (s *Store) ReadVolumeNeedleMetaAt(i needle.VolumeId, n *needle.Needle, offset int64, size int32) error {
|
|
|
|
if v := s.findVolume(i); v != nil {
|
|
|
|
return v.readNeedleMetaAt(n, offset, size)
|
|
|
|
}
|
|
|
|
return fmt.Errorf("volume %d not found", i)
|
|
|
|
}
|
|
|
|
|
2022-06-06 02:54:04 +08:00
|
|
|
func (s *Store) ReadVolumeNeedleDataInto(i needle.VolumeId, n *needle.Needle, readOption *ReadOption, writer io.Writer, offset int64, size int64) error {
|
|
|
|
if v := s.findVolume(i); v != nil {
|
|
|
|
return v.readNeedleDataInto(n, readOption, writer, offset, size)
|
|
|
|
}
|
|
|
|
return fmt.Errorf("volume %d not found", i)
|
|
|
|
}
|
2019-04-19 12:43:36 +08:00
|
|
|
func (s *Store) GetVolume(i needle.VolumeId) *Volume {
|
2013-07-14 02:38:01 +08:00
|
|
|
return s.findVolume(i)
|
2012-09-21 08:58:29 +08:00
|
|
|
}
|
2012-09-11 08:08:52 +08:00
|
|
|
|
2019-04-19 12:43:36 +08:00
|
|
|
func (s *Store) HasVolume(i needle.VolumeId) bool {
|
2013-07-14 02:38:01 +08:00
|
|
|
v := s.findVolume(i)
|
|
|
|
return v != nil
|
2012-09-20 17:47:32 +08:00
|
|
|
}
|
2017-01-20 19:49:20 +08:00
|
|
|
|
2019-06-28 03:18:59 +08:00
|
|
|
func (s *Store) MarkVolumeReadonly(i needle.VolumeId) error {
|
2019-06-27 14:02:22 +08:00
|
|
|
v := s.findVolume(i)
|
|
|
|
if v == nil {
|
|
|
|
return fmt.Errorf("volume %d not found", i)
|
|
|
|
}
|
2020-08-19 23:42:56 +08:00
|
|
|
v.noWriteLock.Lock()
|
2019-12-19 16:42:46 +08:00
|
|
|
v.noWriteOrDelete = true
|
2020-08-19 23:42:56 +08:00
|
|
|
v.noWriteLock.Unlock()
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *Store) MarkVolumeWritable(i needle.VolumeId) error {
|
|
|
|
v := s.findVolume(i)
|
|
|
|
if v == nil {
|
|
|
|
return fmt.Errorf("volume %d not found", i)
|
|
|
|
}
|
|
|
|
v.noWriteLock.Lock()
|
|
|
|
v.noWriteOrDelete = false
|
|
|
|
v.noWriteLock.Unlock()
|
2019-06-27 14:02:22 +08:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-04-19 12:43:36 +08:00
|
|
|
func (s *Store) MountVolume(i needle.VolumeId) error {
|
2017-01-20 19:49:20 +08:00
|
|
|
for _, location := range s.Locations {
|
2021-02-07 09:00:03 +08:00
|
|
|
if found := location.LoadVolume(i, s.NeedleMapKind); found == true {
|
2019-04-21 02:35:20 +08:00
|
|
|
glog.V(0).Infof("mount volume %d", i)
|
|
|
|
v := s.findVolume(i)
|
|
|
|
s.NewVolumesChan <- master_pb.VolumeShortInformationMessage{
|
|
|
|
Id: uint32(v.Id),
|
|
|
|
Collection: v.Collection,
|
|
|
|
ReplicaPlacement: uint32(v.ReplicaPlacement.Byte()),
|
2019-04-21 14:53:37 +08:00
|
|
|
Version: uint32(v.Version()),
|
2019-04-21 02:35:20 +08:00
|
|
|
Ttl: v.Ttl.ToUint32(),
|
2020-12-17 01:10:14 +08:00
|
|
|
DiskType: string(v.location.DiskType),
|
2019-04-21 02:35:20 +08:00
|
|
|
}
|
2017-01-20 19:49:20 +08:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-05-26 16:14:42 +08:00
|
|
|
return fmt.Errorf("volume %d not found on disk", i)
|
2017-01-20 19:49:20 +08:00
|
|
|
}
|
|
|
|
|
2019-04-19 12:43:36 +08:00
|
|
|
func (s *Store) UnmountVolume(i needle.VolumeId) error {
|
2019-04-21 02:35:20 +08:00
|
|
|
v := s.findVolume(i)
|
|
|
|
if v == nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
message := master_pb.VolumeShortInformationMessage{
|
|
|
|
Id: uint32(v.Id),
|
|
|
|
Collection: v.Collection,
|
|
|
|
ReplicaPlacement: uint32(v.ReplicaPlacement.Byte()),
|
2019-04-21 14:53:37 +08:00
|
|
|
Version: uint32(v.Version()),
|
2019-04-21 02:35:20 +08:00
|
|
|
Ttl: v.Ttl.ToUint32(),
|
2020-12-17 01:10:14 +08:00
|
|
|
DiskType: string(v.location.DiskType),
|
2019-04-21 02:35:20 +08:00
|
|
|
}
|
2019-05-26 14:23:19 +08:00
|
|
|
|
2017-01-20 19:49:20 +08:00
|
|
|
for _, location := range s.Locations {
|
2022-04-20 13:58:57 +08:00
|
|
|
err := location.UnloadVolume(i)
|
|
|
|
if err == nil {
|
2019-04-21 02:35:20 +08:00
|
|
|
glog.V(0).Infof("UnmountVolume %d", i)
|
2022-07-27 19:31:49 +08:00
|
|
|
stats.DeleteCollectionMetrics(v.Collection)
|
2019-04-21 02:35:20 +08:00
|
|
|
s.DeletedVolumesChan <- message
|
2017-01-20 19:49:20 +08:00
|
|
|
return nil
|
2022-04-20 13:58:57 +08:00
|
|
|
} else if err == ErrVolumeNotFound {
|
|
|
|
continue
|
2017-01-20 19:49:20 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-05-26 16:14:42 +08:00
|
|
|
return fmt.Errorf("volume %d not found on disk", i)
|
2017-01-20 19:49:20 +08:00
|
|
|
}
|
2018-12-29 16:03:30 +08:00
|
|
|
|
2023-06-13 01:42:44 +08:00
|
|
|
func (s *Store) DeleteVolume(i needle.VolumeId, onlyEmpty bool) error {
|
2019-04-21 02:35:20 +08:00
|
|
|
v := s.findVolume(i)
|
|
|
|
if v == nil {
|
2020-04-22 05:49:58 +08:00
|
|
|
return fmt.Errorf("delete volume %d not found on disk", i)
|
2019-04-21 02:35:20 +08:00
|
|
|
}
|
|
|
|
message := master_pb.VolumeShortInformationMessage{
|
|
|
|
Id: uint32(v.Id),
|
|
|
|
Collection: v.Collection,
|
|
|
|
ReplicaPlacement: uint32(v.ReplicaPlacement.Byte()),
|
2019-04-21 14:53:37 +08:00
|
|
|
Version: uint32(v.Version()),
|
2019-04-21 02:35:20 +08:00
|
|
|
Ttl: v.Ttl.ToUint32(),
|
2020-12-17 01:10:14 +08:00
|
|
|
DiskType: string(v.location.DiskType),
|
2019-04-21 02:35:20 +08:00
|
|
|
}
|
2018-12-29 16:03:30 +08:00
|
|
|
for _, location := range s.Locations {
|
2023-06-15 05:39:58 +08:00
|
|
|
err := location.DeleteVolume(i, onlyEmpty)
|
2022-04-20 13:58:57 +08:00
|
|
|
if err == nil {
|
2019-04-21 02:35:20 +08:00
|
|
|
glog.V(0).Infof("DeleteVolume %d", i)
|
|
|
|
s.DeletedVolumesChan <- message
|
2018-12-29 16:03:30 +08:00
|
|
|
return nil
|
2022-04-20 13:58:57 +08:00
|
|
|
} else if err == ErrVolumeNotFound {
|
|
|
|
continue
|
2023-06-15 05:39:58 +08:00
|
|
|
} else if err == ErrVolumeNotEmpty {
|
|
|
|
return fmt.Errorf("DeleteVolume %d: %v", i, err)
|
2020-10-01 22:10:03 +08:00
|
|
|
} else {
|
|
|
|
glog.Errorf("DeleteVolume %d: %v", i, err)
|
2018-12-29 16:03:30 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-05-26 16:14:42 +08:00
|
|
|
return fmt.Errorf("volume %d not found on disk", i)
|
2018-12-29 16:03:30 +08:00
|
|
|
}
|
2019-04-15 14:00:37 +08:00
|
|
|
|
2020-02-03 07:37:23 +08:00
|
|
|
func (s *Store) ConfigureVolume(i needle.VolumeId, replication string) error {
|
|
|
|
|
|
|
|
for _, location := range s.Locations {
|
|
|
|
fileInfo, found := location.LocateVolume(i)
|
|
|
|
if !found {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
// load, modify, save
|
|
|
|
baseFileName := strings.TrimSuffix(fileInfo.Name(), filepath.Ext(fileInfo.Name()))
|
2020-02-27 08:52:57 +08:00
|
|
|
vifFile := filepath.Join(location.Directory, baseFileName+".vif")
|
2021-08-27 06:18:34 +08:00
|
|
|
volumeInfo, _, _, err := volume_info.MaybeLoadVolumeInfo(vifFile)
|
2020-02-03 07:37:23 +08:00
|
|
|
if err != nil {
|
2022-10-14 15:18:09 +08:00
|
|
|
return fmt.Errorf("volume %d failed to load vif: %v", i, err)
|
2020-02-03 07:37:23 +08:00
|
|
|
}
|
|
|
|
volumeInfo.Replication = replication
|
2021-08-27 06:18:34 +08:00
|
|
|
err = volume_info.SaveVolumeInfo(vifFile, volumeInfo)
|
2020-02-03 07:37:23 +08:00
|
|
|
if err != nil {
|
2022-10-14 15:18:09 +08:00
|
|
|
return fmt.Errorf("volume %d failed to save vif: %v", i, err)
|
2020-02-03 07:37:23 +08:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return fmt.Errorf("volume %d not found on disk", i)
|
|
|
|
}
|
|
|
|
|
2019-04-15 14:00:37 +08:00
|
|
|
func (s *Store) SetVolumeSizeLimit(x uint64) {
|
|
|
|
atomic.StoreUint64(&s.volumeSizeLimit, x)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *Store) GetVolumeSizeLimit() uint64 {
|
|
|
|
return atomic.LoadUint64(&s.volumeSizeLimit)
|
|
|
|
}
|
2020-03-23 07:21:42 +08:00
|
|
|
|
|
|
|
func (s *Store) MaybeAdjustVolumeMax() (hasChanges bool) {
|
|
|
|
volumeSizeLimit := s.GetVolumeSizeLimit()
|
2021-02-16 21:59:24 +08:00
|
|
|
if volumeSizeLimit == 0 {
|
|
|
|
return
|
|
|
|
}
|
2023-07-11 14:37:33 +08:00
|
|
|
var newMaxVolumeCount int32
|
2020-03-23 07:21:42 +08:00
|
|
|
for _, diskLocation := range s.Locations {
|
2020-11-04 06:43:17 +08:00
|
|
|
if diskLocation.OriginalMaxVolumeCount == 0 {
|
2022-08-26 23:41:42 +08:00
|
|
|
currentMaxVolumeCount := atomic.LoadInt32(&diskLocation.MaxVolumeCount)
|
2020-03-23 07:21:42 +08:00
|
|
|
diskStatus := stats.NewDiskStatus(diskLocation.Directory)
|
|
|
|
unusedSpace := diskLocation.UnUsedSpace(volumeSizeLimit)
|
|
|
|
unclaimedSpaces := int64(diskStatus.Free) - int64(unusedSpace)
|
|
|
|
volCount := diskLocation.VolumesLen()
|
2022-08-26 23:41:42 +08:00
|
|
|
maxVolumeCount := int32(volCount)
|
2020-03-23 07:21:42 +08:00
|
|
|
if unclaimedSpaces > int64(volumeSizeLimit) {
|
2022-08-26 23:41:42 +08:00
|
|
|
maxVolumeCount += int32(uint64(unclaimedSpaces)/volumeSizeLimit) - 1
|
2020-03-23 07:21:42 +08:00
|
|
|
}
|
2023-07-11 14:37:33 +08:00
|
|
|
newMaxVolumeCount = newMaxVolumeCount + maxVolumeCount
|
2022-08-26 23:41:42 +08:00
|
|
|
atomic.StoreInt32(&diskLocation.MaxVolumeCount, maxVolumeCount)
|
2022-09-05 13:21:24 +08:00
|
|
|
glog.V(4).Infof("disk %s max %d unclaimedSpace:%dMB, unused:%dMB volumeSizeLimit:%dMB",
|
2020-03-23 07:21:42 +08:00
|
|
|
diskLocation.Directory, maxVolumeCount, unclaimedSpaces/1024/1024, unusedSpace/1024/1024, volumeSizeLimit/1024/1024)
|
2022-08-26 23:41:42 +08:00
|
|
|
hasChanges = hasChanges || currentMaxVolumeCount != atomic.LoadInt32(&diskLocation.MaxVolumeCount)
|
2023-07-11 14:37:33 +08:00
|
|
|
} else {
|
|
|
|
newMaxVolumeCount = newMaxVolumeCount + diskLocation.OriginalMaxVolumeCount
|
2020-03-23 07:21:42 +08:00
|
|
|
}
|
|
|
|
}
|
2023-07-11 14:37:33 +08:00
|
|
|
stats.VolumeServerMaxVolumeCounter.Set(float64(newMaxVolumeCount))
|
2020-03-23 07:21:42 +08:00
|
|
|
return
|
|
|
|
}
|