2011-12-16 22:51:26 +08:00
|
|
|
package storage
|
|
|
|
|
|
|
|
import (
|
2012-11-20 17:45:36 +08:00
|
|
|
"fmt"
|
2019-12-19 16:42:46 +08:00
|
|
|
"path"
|
|
|
|
"strconv"
|
|
|
|
"sync"
|
|
|
|
"time"
|
2019-04-19 12:43:36 +08:00
|
|
|
|
2019-09-12 21:18:21 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
|
2019-12-03 07:08:28 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
2019-09-12 21:18:21 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/stats"
|
2019-10-29 15:35:16 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/storage/backend"
|
2019-09-12 21:18:21 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
2019-12-24 04:48:20 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
|
2019-09-12 21:18:21 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
2019-04-19 12:43:36 +08:00
|
|
|
|
2019-09-12 21:18:21 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
2011-12-16 22:51:26 +08:00
|
|
|
)
|
|
|
|
|
|
|
|
type Volume struct {
|
2019-09-05 18:56:48 +08:00
|
|
|
Id needle.VolumeId
|
|
|
|
dir string
|
|
|
|
Collection string
|
2019-11-29 10:33:18 +08:00
|
|
|
DataBackend backend.BackendStorageFile
|
2019-09-05 18:56:48 +08:00
|
|
|
nm NeedleMapper
|
|
|
|
needleMapKind NeedleMapType
|
2019-12-19 16:42:46 +08:00
|
|
|
noWriteOrDelete bool // if readonly, either noWriteOrDelete or noWriteCanDelete
|
|
|
|
noWriteCanDelete bool // if readonly, either noWriteOrDelete or noWriteCanDelete
|
2019-12-29 03:35:27 +08:00
|
|
|
hasRemoteFile bool // if the volume has a remote file
|
2019-10-22 13:57:01 +08:00
|
|
|
MemoryMapMaxSizeMb uint32
|
2011-12-16 22:51:26 +08:00
|
|
|
|
2019-12-24 04:48:20 +08:00
|
|
|
super_block.SuperBlock
|
2012-11-20 17:45:36 +08:00
|
|
|
|
2019-12-06 22:59:57 +08:00
|
|
|
dataFileAccessLock sync.RWMutex
|
2019-04-19 15:39:34 +08:00
|
|
|
lastModifiedTsSeconds uint64 //unix time in seconds
|
|
|
|
lastAppendAtNs uint64 //unix time in nanoseconds
|
2016-09-29 13:57:23 +08:00
|
|
|
|
2016-10-07 16:22:24 +08:00
|
|
|
lastCompactIndexOffset uint64
|
|
|
|
lastCompactRevision uint16
|
2019-08-12 15:53:50 +08:00
|
|
|
|
|
|
|
isCompacting bool
|
2019-12-03 07:08:28 +08:00
|
|
|
|
2019-12-29 03:21:49 +08:00
|
|
|
volumeInfo *volume_server_pb.VolumeInfo
|
2011-12-16 22:51:26 +08:00
|
|
|
}
|
|
|
|
|
2019-12-24 04:48:20 +08:00
|
|
|
func NewVolume(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *super_block.ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapMaxSizeMb uint32) (v *Volume, e error) {
|
2018-06-24 07:48:19 +08:00
|
|
|
// if replicaPlacement is nil, the superblock will be loaded from disk
|
2019-10-30 14:16:43 +08:00
|
|
|
v = &Volume{dir: dirname, Collection: collection, Id: id, MemoryMapMaxSizeMb: memoryMapMaxSizeMb}
|
2019-12-24 04:48:20 +08:00
|
|
|
v.SuperBlock = super_block.SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl}
|
Add boltdb for volume needle map
boltdb is fairly slow to write, about 6 minutes for recreating index
for 1553934 files. Boltdb loads 1,553,934 x 16 = 24,862,944bytes from
disk, and generate the boltdb as large as 134,217,728 bytes in 6
minutes.
To compare, for leveldb, it recreates index in leveldb as large as
27,188,148 bytes in 8 seconds.
For in memory version, it loads the index in
To test the memory consumption, the leveldb or boltdb index are
created. And the server is restarted. Using the benchmark tool to read
lots of files. There are 7 volumes in benchmark collection, each with
about 1553K files.
For leveldb, the memory starts at 142,884KB, and stays at 179,340KB.
For boltdb, the memory starts at 73,756KB, and stays at 144,564KB.
For in-memory, the memory starts at 368,152KB, and stays at 448,032KB.
2015-03-30 02:04:32 +08:00
|
|
|
v.needleMapKind = needleMapKind
|
2019-09-04 00:00:59 +08:00
|
|
|
e = v.load(true, true, needleMapKind, preallocate)
|
2013-01-21 11:44:23 +08:00
|
|
|
return
|
|
|
|
}
|
2014-12-26 15:36:33 +08:00
|
|
|
func (v *Volume) String() string {
|
2019-12-19 16:42:46 +08:00
|
|
|
return fmt.Sprintf("Id:%v, dir:%s, Collection:%s, dataFile:%v, nm:%v, noWrite:%v canDelete:%v", v.Id, v.dir, v.Collection, v.DataBackend, v.nm, v.noWriteOrDelete || v.noWriteCanDelete, v.noWriteCanDelete)
|
2014-12-26 15:36:33 +08:00
|
|
|
}
|
|
|
|
|
2019-06-03 17:26:31 +08:00
|
|
|
func VolumeFileName(dir string, collection string, id int) (fileName string) {
|
2019-03-24 02:33:34 +08:00
|
|
|
idString := strconv.Itoa(id)
|
|
|
|
if collection == "" {
|
|
|
|
fileName = path.Join(dir, idString)
|
2014-02-07 09:32:06 +08:00
|
|
|
} else {
|
2019-03-24 02:33:34 +08:00
|
|
|
fileName = path.Join(dir, collection+"_"+idString)
|
2014-02-07 09:32:06 +08:00
|
|
|
}
|
|
|
|
return
|
2014-01-22 12:51:46 +08:00
|
|
|
}
|
2019-03-24 02:33:34 +08:00
|
|
|
func (v *Volume) FileName() (fileName string) {
|
2019-06-03 17:26:31 +08:00
|
|
|
return VolumeFileName(v.dir, v.Collection, int(v.Id))
|
2019-03-24 02:33:34 +08:00
|
|
|
}
|
2014-02-07 09:32:06 +08:00
|
|
|
|
2019-04-19 12:43:36 +08:00
|
|
|
func (v *Volume) Version() needle.Version {
|
2019-12-29 04:28:58 +08:00
|
|
|
if v.volumeInfo.Version != 0 {
|
|
|
|
v.SuperBlock.Version = needle.Version(v.volumeInfo.Version)
|
|
|
|
}
|
2019-12-24 04:48:20 +08:00
|
|
|
return v.SuperBlock.Version
|
2012-12-18 09:51:39 +08:00
|
|
|
}
|
2016-07-03 14:56:49 +08:00
|
|
|
|
2019-04-19 15:39:34 +08:00
|
|
|
func (v *Volume) FileStat() (datSize uint64, idxSize uint64, modTime time.Time) {
|
2019-12-06 22:59:57 +08:00
|
|
|
v.dataFileAccessLock.RLock()
|
|
|
|
defer v.dataFileAccessLock.RUnlock()
|
2018-11-04 15:28:24 +08:00
|
|
|
|
2019-10-29 15:35:16 +08:00
|
|
|
if v.DataBackend == nil {
|
2019-04-19 15:39:34 +08:00
|
|
|
return
|
2018-11-04 15:28:24 +08:00
|
|
|
}
|
|
|
|
|
2019-10-29 15:35:16 +08:00
|
|
|
datFileSize, modTime, e := v.DataBackend.GetStat()
|
2011-12-22 12:04:47 +08:00
|
|
|
if e == nil {
|
2019-10-29 15:35:16 +08:00
|
|
|
return uint64(datFileSize), v.nm.IndexFileSize(), modTime
|
2011-12-22 12:04:47 +08:00
|
|
|
}
|
2019-12-09 11:44:16 +08:00
|
|
|
glog.V(0).Infof("Failed to read file size %s %v", v.DataBackend.Name(), e)
|
2019-04-19 15:39:34 +08:00
|
|
|
return // -1 causes integer overflow and the volume to become unwritable.
|
2011-12-17 14:47:23 +08:00
|
|
|
}
|
2015-03-09 16:09:15 +08:00
|
|
|
|
2019-08-14 16:08:01 +08:00
|
|
|
func (v *Volume) ContentSize() uint64 {
|
2019-12-06 22:59:57 +08:00
|
|
|
v.dataFileAccessLock.RLock()
|
|
|
|
defer v.dataFileAccessLock.RUnlock()
|
2019-09-10 00:48:08 +08:00
|
|
|
if v.nm == nil {
|
|
|
|
return 0
|
|
|
|
}
|
2019-08-14 16:08:01 +08:00
|
|
|
return v.nm.ContentSize()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (v *Volume) DeletedSize() uint64 {
|
2019-12-06 22:59:57 +08:00
|
|
|
v.dataFileAccessLock.RLock()
|
|
|
|
defer v.dataFileAccessLock.RUnlock()
|
2019-09-10 00:48:08 +08:00
|
|
|
if v.nm == nil {
|
|
|
|
return 0
|
|
|
|
}
|
2019-08-14 16:08:01 +08:00
|
|
|
return v.nm.DeletedSize()
|
2019-04-10 19:41:55 +08:00
|
|
|
}
|
|
|
|
|
2019-04-18 15:19:18 +08:00
|
|
|
func (v *Volume) FileCount() uint64 {
|
2019-12-06 22:59:57 +08:00
|
|
|
v.dataFileAccessLock.RLock()
|
|
|
|
defer v.dataFileAccessLock.RUnlock()
|
2019-09-10 00:48:08 +08:00
|
|
|
if v.nm == nil {
|
|
|
|
return 0
|
|
|
|
}
|
2019-04-11 14:39:53 +08:00
|
|
|
return uint64(v.nm.FileCount())
|
2019-04-10 19:41:55 +08:00
|
|
|
}
|
|
|
|
|
2019-08-14 16:08:01 +08:00
|
|
|
func (v *Volume) DeletedCount() uint64 {
|
2019-12-06 22:59:57 +08:00
|
|
|
v.dataFileAccessLock.RLock()
|
|
|
|
defer v.dataFileAccessLock.RUnlock()
|
2019-09-10 00:48:08 +08:00
|
|
|
if v.nm == nil {
|
|
|
|
return 0
|
|
|
|
}
|
2019-08-14 16:08:01 +08:00
|
|
|
return uint64(v.nm.DeletedCount())
|
|
|
|
}
|
|
|
|
|
|
|
|
func (v *Volume) MaxFileKey() types.NeedleId {
|
2019-12-06 22:59:57 +08:00
|
|
|
v.dataFileAccessLock.RLock()
|
|
|
|
defer v.dataFileAccessLock.RUnlock()
|
2019-09-10 00:48:08 +08:00
|
|
|
if v.nm == nil {
|
|
|
|
return 0
|
|
|
|
}
|
2019-08-14 16:08:01 +08:00
|
|
|
return v.nm.MaxFileKey()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (v *Volume) IndexFileSize() uint64 {
|
2019-12-06 22:59:57 +08:00
|
|
|
v.dataFileAccessLock.RLock()
|
|
|
|
defer v.dataFileAccessLock.RUnlock()
|
2019-09-10 00:48:08 +08:00
|
|
|
if v.nm == nil {
|
|
|
|
return 0
|
|
|
|
}
|
2019-08-14 16:08:01 +08:00
|
|
|
return v.nm.IndexFileSize()
|
|
|
|
}
|
|
|
|
|
2015-03-09 16:09:15 +08:00
|
|
|
// Close cleanly shuts down this volume
|
2011-12-16 22:51:26 +08:00
|
|
|
func (v *Volume) Close() {
|
2015-05-24 01:16:01 +08:00
|
|
|
v.dataFileAccessLock.Lock()
|
|
|
|
defer v.dataFileAccessLock.Unlock()
|
2018-11-04 15:28:24 +08:00
|
|
|
if v.nm != nil {
|
|
|
|
v.nm.Close()
|
|
|
|
v.nm = nil
|
|
|
|
}
|
2019-10-29 15:35:16 +08:00
|
|
|
if v.DataBackend != nil {
|
|
|
|
_ = v.DataBackend.Close()
|
|
|
|
v.DataBackend = nil
|
2019-06-18 12:02:50 +08:00
|
|
|
stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume").Dec()
|
2018-11-04 15:28:24 +08:00
|
|
|
}
|
2011-12-16 22:51:26 +08:00
|
|
|
}
|
2015-03-09 16:09:15 +08:00
|
|
|
|
2012-11-20 16:54:37 +08:00
|
|
|
func (v *Volume) NeedToReplicate() bool {
|
2014-03-03 14:16:54 +08:00
|
|
|
return v.ReplicaPlacement.GetCopyCount() > 1
|
2012-11-12 17:26:18 +08:00
|
|
|
}
|
2011-12-16 22:51:26 +08:00
|
|
|
|
2014-09-21 03:38:59 +08:00
|
|
|
// volume is expired if modified time + volume ttl < now
|
|
|
|
// except when volume is empty
|
|
|
|
// or when the volume does not have a ttl
|
|
|
|
// or when volumeSizeLimit is 0 when server just starts
|
|
|
|
func (v *Volume) expired(volumeSizeLimit uint64) bool {
|
|
|
|
if volumeSizeLimit == 0 {
|
|
|
|
//skip if we don't know size limit
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if v.ContentSize() == 0 {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if v.Ttl == nil || v.Ttl.Minutes() == 0 {
|
|
|
|
return false
|
|
|
|
}
|
2019-04-19 15:39:34 +08:00
|
|
|
glog.V(1).Infof("now:%v lastModified:%v", time.Now().Unix(), v.lastModifiedTsSeconds)
|
|
|
|
livedMinutes := (time.Now().Unix() - int64(v.lastModifiedTsSeconds)) / 60
|
2016-11-14 06:07:51 +08:00
|
|
|
glog.V(1).Infof("ttl:%v lived:%v", v.Ttl, livedMinutes)
|
2014-09-21 03:38:59 +08:00
|
|
|
if int64(v.Ttl.Minutes()) < livedMinutes {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// wait either maxDelayMinutes or 10% of ttl minutes
|
2019-01-17 09:17:19 +08:00
|
|
|
func (v *Volume) expiredLongEnough(maxDelayMinutes uint32) bool {
|
2014-09-21 03:38:59 +08:00
|
|
|
if v.Ttl == nil || v.Ttl.Minutes() == 0 {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
removalDelay := v.Ttl.Minutes() / 10
|
|
|
|
if removalDelay > maxDelayMinutes {
|
|
|
|
removalDelay = maxDelayMinutes
|
|
|
|
}
|
|
|
|
|
2019-04-19 15:39:34 +08:00
|
|
|
if uint64(v.Ttl.Minutes()+removalDelay)*60+v.lastModifiedTsSeconds < uint64(time.Now().Unix()) {
|
2014-09-21 03:38:59 +08:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
2019-03-18 11:27:08 +08:00
|
|
|
|
|
|
|
func (v *Volume) ToVolumeInformationMessage() *master_pb.VolumeInformationMessage {
|
2019-06-01 14:41:17 +08:00
|
|
|
size, _, modTime := v.FileStat()
|
2019-08-14 16:08:01 +08:00
|
|
|
|
2019-12-03 15:23:54 +08:00
|
|
|
volumInfo := &master_pb.VolumeInformationMessage{
|
2019-03-18 11:27:08 +08:00
|
|
|
Id: uint32(v.Id),
|
2019-04-19 15:39:34 +08:00
|
|
|
Size: size,
|
2019-03-18 11:27:08 +08:00
|
|
|
Collection: v.Collection,
|
2019-12-25 10:13:45 +08:00
|
|
|
FileCount: v.FileCount(),
|
|
|
|
DeleteCount: v.DeletedCount(),
|
2019-08-14 16:08:01 +08:00
|
|
|
DeletedByteCount: v.DeletedSize(),
|
2020-03-02 09:57:37 +08:00
|
|
|
ReadOnly: v.noWriteOrDelete || v.noWriteCanDelete,
|
2019-03-18 11:27:08 +08:00
|
|
|
ReplicaPlacement: uint32(v.ReplicaPlacement.Byte()),
|
|
|
|
Version: uint32(v.Version()),
|
|
|
|
Ttl: v.Ttl.ToUint32(),
|
2019-04-20 03:29:49 +08:00
|
|
|
CompactRevision: uint32(v.SuperBlock.CompactionRevision),
|
2019-06-01 14:41:17 +08:00
|
|
|
ModifiedAtSecond: modTime.Unix(),
|
2019-03-18 11:27:08 +08:00
|
|
|
}
|
2019-12-03 15:23:54 +08:00
|
|
|
|
|
|
|
volumInfo.RemoteStorageName, volumInfo.RemoteStorageKey = v.RemoteStorageNameKey()
|
|
|
|
|
|
|
|
return volumInfo
|
|
|
|
}
|
|
|
|
|
|
|
|
func (v *Volume) RemoteStorageNameKey() (storageName, storageKey string) {
|
2019-12-29 03:21:49 +08:00
|
|
|
if v.volumeInfo == nil {
|
2019-12-26 08:17:58 +08:00
|
|
|
return
|
|
|
|
}
|
2019-12-29 03:21:49 +08:00
|
|
|
if len(v.volumeInfo.GetFiles()) == 0 {
|
2019-12-03 15:23:54 +08:00
|
|
|
return
|
|
|
|
}
|
2019-12-29 03:21:49 +08:00
|
|
|
return v.volumeInfo.GetFiles()[0].BackendName(), v.volumeInfo.GetFiles()[0].GetKey()
|
2019-03-18 11:27:08 +08:00
|
|
|
}
|