2012-11-07 17:51:43 +08:00
|
|
|
package topology
|
|
|
|
|
|
|
|
import (
|
2018-10-15 14:12:43 +08:00
|
|
|
"context"
|
2019-07-22 12:49:10 +08:00
|
|
|
"sync/atomic"
|
2012-11-07 17:51:43 +08:00
|
|
|
"time"
|
2014-10-27 02:34:55 +08:00
|
|
|
|
2019-04-19 12:43:36 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
|
|
|
"google.golang.org/grpc"
|
|
|
|
|
2016-06-03 09:09:14 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
2018-10-15 14:12:43 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/operation"
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
2012-11-07 17:51:43 +08:00
|
|
|
)
|
|
|
|
|
2019-12-01 23:29:41 +08:00
|
|
|
func batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId,
|
|
|
|
locationlist *VolumeLocationList, garbageThreshold float64) (*VolumeLocationList, bool) {
|
|
|
|
ch := make(chan int, locationlist.Length())
|
|
|
|
errCount := int32(0)
|
2012-11-24 09:03:27 +08:00
|
|
|
for index, dn := range locationlist.list {
|
2019-04-19 12:43:36 +08:00
|
|
|
go func(index int, url string, vid needle.VolumeId) {
|
2020-02-26 13:50:12 +08:00
|
|
|
err := operation.WithVolumeServerClient(url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
|
|
|
|
resp, err := volumeServerClient.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
|
2019-03-24 02:33:34 +08:00
|
|
|
VolumeId: uint32(vid),
|
2018-10-15 14:12:43 +08:00
|
|
|
})
|
|
|
|
if err != nil {
|
2019-12-01 23:29:41 +08:00
|
|
|
atomic.AddInt32(&errCount, 1)
|
|
|
|
ch <- -1
|
2018-10-15 14:12:43 +08:00
|
|
|
return err
|
|
|
|
}
|
2019-12-01 23:29:41 +08:00
|
|
|
if resp.GarbageRatio >= garbageThreshold {
|
|
|
|
ch <- index
|
|
|
|
} else {
|
|
|
|
ch <- -1
|
|
|
|
}
|
2018-10-15 14:12:43 +08:00
|
|
|
return nil
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
glog.V(0).Infof("Checking vacuuming %d on %s: %v", vid, url, err)
|
2012-11-24 09:03:27 +08:00
|
|
|
}
|
2015-03-09 16:10:01 +08:00
|
|
|
}(index, dn.Url(), vid)
|
2012-11-24 09:03:27 +08:00
|
|
|
}
|
2019-12-01 23:29:41 +08:00
|
|
|
vacuumLocationList := NewVolumeLocationList()
|
2019-11-20 08:24:58 +08:00
|
|
|
for range locationlist.list {
|
2012-11-24 09:03:27 +08:00
|
|
|
select {
|
2019-12-01 23:29:41 +08:00
|
|
|
case index := <-ch:
|
|
|
|
if index != -1 {
|
|
|
|
vacuumLocationList.list = append(vacuumLocationList.list, locationlist.list[index])
|
|
|
|
}
|
2012-11-24 09:03:27 +08:00
|
|
|
case <-time.After(30 * time.Minute):
|
2019-12-01 23:29:41 +08:00
|
|
|
return vacuumLocationList, false
|
2012-11-24 09:03:27 +08:00
|
|
|
}
|
|
|
|
}
|
2019-12-01 23:29:41 +08:00
|
|
|
return vacuumLocationList, errCount == 0 && len(vacuumLocationList.list) > 0
|
2012-11-24 09:03:27 +08:00
|
|
|
}
|
2019-12-01 23:29:41 +08:00
|
|
|
func batchVacuumVolumeCompact(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId,
|
|
|
|
locationlist *VolumeLocationList, preallocate int64) bool {
|
2019-07-22 04:49:09 +08:00
|
|
|
vl.accessLock.Lock()
|
2012-11-24 09:03:27 +08:00
|
|
|
vl.removeFromWritable(vid)
|
2019-07-22 04:49:09 +08:00
|
|
|
vl.accessLock.Unlock()
|
|
|
|
|
2012-11-24 09:03:27 +08:00
|
|
|
ch := make(chan bool, locationlist.Length())
|
|
|
|
for index, dn := range locationlist.list {
|
2019-04-19 12:43:36 +08:00
|
|
|
go func(index int, url string, vid needle.VolumeId) {
|
2013-08-09 14:57:22 +08:00
|
|
|
glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url)
|
2020-02-26 13:50:12 +08:00
|
|
|
err := operation.WithVolumeServerClient(url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
|
|
|
|
_, err := volumeServerClient.VacuumVolumeCompact(context.Background(), &volume_server_pb.VacuumVolumeCompactRequest{
|
2019-03-24 02:33:34 +08:00
|
|
|
VolumeId: uint32(vid),
|
2018-10-15 14:12:43 +08:00
|
|
|
})
|
|
|
|
return err
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
glog.Errorf("Error when vacuuming %d on %s: %v", vid, url, err)
|
2012-11-24 09:03:27 +08:00
|
|
|
ch <- false
|
|
|
|
} else {
|
2018-10-15 14:12:43 +08:00
|
|
|
glog.V(0).Infof("Complete vacuuming %d on %s", vid, url)
|
2012-11-24 09:03:27 +08:00
|
|
|
ch <- true
|
|
|
|
}
|
2015-03-09 16:10:01 +08:00
|
|
|
}(index, dn.Url(), vid)
|
2012-11-24 09:03:27 +08:00
|
|
|
}
|
|
|
|
isVacuumSuccess := true
|
2019-11-20 08:24:58 +08:00
|
|
|
for range locationlist.list {
|
2012-11-24 09:03:27 +08:00
|
|
|
select {
|
2017-08-30 14:11:08 +08:00
|
|
|
case canCommit := <-ch:
|
|
|
|
isVacuumSuccess = isVacuumSuccess && canCommit
|
2012-11-24 09:03:27 +08:00
|
|
|
case <-time.After(30 * time.Minute):
|
2019-11-20 08:24:58 +08:00
|
|
|
return false
|
2012-11-24 09:03:27 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return isVacuumSuccess
|
|
|
|
}
|
2019-04-19 12:43:36 +08:00
|
|
|
func batchVacuumVolumeCommit(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) bool {
|
2012-11-24 09:03:27 +08:00
|
|
|
isCommitSuccess := true
|
|
|
|
for _, dn := range locationlist.list {
|
2019-02-06 21:59:15 +08:00
|
|
|
glog.V(0).Infoln("Start Committing vacuum", vid, "on", dn.Url())
|
2020-02-26 13:50:12 +08:00
|
|
|
err := operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
|
|
|
|
_, err := volumeServerClient.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{
|
2019-03-24 02:33:34 +08:00
|
|
|
VolumeId: uint32(vid),
|
2018-10-15 14:12:43 +08:00
|
|
|
})
|
|
|
|
return err
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
glog.Errorf("Error when committing vacuum %d on %s: %v", vid, dn.Url(), err)
|
2012-11-24 09:03:27 +08:00
|
|
|
isCommitSuccess = false
|
|
|
|
} else {
|
2019-02-06 21:59:15 +08:00
|
|
|
glog.V(0).Infof("Complete Committing vacuum %d on %s", vid, dn.Url())
|
2012-11-24 09:03:27 +08:00
|
|
|
}
|
2013-10-16 23:10:48 +08:00
|
|
|
if isCommitSuccess {
|
|
|
|
vl.SetVolumeAvailable(dn, vid)
|
|
|
|
}
|
2012-11-24 09:03:27 +08:00
|
|
|
}
|
|
|
|
return isCommitSuccess
|
|
|
|
}
|
2019-04-19 12:43:36 +08:00
|
|
|
func batchVacuumVolumeCleanup(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) {
|
2017-08-30 14:59:53 +08:00
|
|
|
for _, dn := range locationlist.list {
|
|
|
|
glog.V(0).Infoln("Start cleaning up", vid, "on", dn.Url())
|
2020-02-26 13:50:12 +08:00
|
|
|
err := operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
|
|
|
|
_, err := volumeServerClient.VacuumVolumeCleanup(context.Background(), &volume_server_pb.VacuumVolumeCleanupRequest{
|
2019-03-24 02:33:34 +08:00
|
|
|
VolumeId: uint32(vid),
|
2018-10-15 14:12:43 +08:00
|
|
|
})
|
|
|
|
return err
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
glog.Errorf("Error when cleaning up vacuum %d on %s: %v", vid, dn.Url(), err)
|
2017-08-30 14:59:53 +08:00
|
|
|
} else {
|
2018-10-15 14:12:43 +08:00
|
|
|
glog.V(0).Infof("Complete cleaning up vacuum %d on %s", vid, dn.Url())
|
2017-08-30 14:59:53 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-02-19 04:11:52 +08:00
|
|
|
func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, preallocate int64) int {
|
2019-07-22 12:49:10 +08:00
|
|
|
|
|
|
|
// if there is vacuum going on, return immediately
|
|
|
|
swapped := atomic.CompareAndSwapInt64(&t.vacuumLockCounter, 0, 1)
|
|
|
|
if !swapped {
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
defer atomic.StoreInt64(&t.vacuumLockCounter, 0)
|
|
|
|
|
|
|
|
// now only one vacuum process going on
|
|
|
|
|
2018-12-31 16:06:52 +08:00
|
|
|
glog.V(1).Infof("Start vacuum on demand with threshold: %f", garbageThreshold)
|
2016-05-31 03:30:26 +08:00
|
|
|
for _, col := range t.collectionMap.Items() {
|
2014-12-09 12:29:25 +08:00
|
|
|
c := col.(*Collection)
|
2016-05-31 03:30:26 +08:00
|
|
|
for _, vl := range c.storageType2VolumeLayout.Items() {
|
2013-11-12 18:21:22 +08:00
|
|
|
if vl != nil {
|
2014-12-09 12:29:25 +08:00
|
|
|
volumeLayout := vl.(*VolumeLayout)
|
2019-02-19 04:11:52 +08:00
|
|
|
vacuumOneVolumeLayout(grpcDialOption, volumeLayout, c, garbageThreshold, preallocate)
|
2018-10-19 11:34:43 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
2019-02-19 04:11:52 +08:00
|
|
|
func vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, preallocate int64) {
|
2017-05-23 08:05:27 +08:00
|
|
|
|
2018-10-19 11:34:43 +08:00
|
|
|
volumeLayout.accessLock.RLock()
|
2019-04-19 12:43:36 +08:00
|
|
|
tmpMap := make(map[needle.VolumeId]*VolumeLocationList)
|
2019-01-17 09:17:19 +08:00
|
|
|
for vid, locationList := range volumeLayout.vid2location {
|
|
|
|
tmpMap[vid] = locationList
|
2018-10-19 11:34:43 +08:00
|
|
|
}
|
|
|
|
volumeLayout.accessLock.RUnlock()
|
2017-05-23 08:05:27 +08:00
|
|
|
|
2019-01-17 09:17:19 +08:00
|
|
|
for vid, locationList := range tmpMap {
|
2017-05-23 08:05:27 +08:00
|
|
|
|
2018-10-19 11:34:43 +08:00
|
|
|
volumeLayout.accessLock.RLock()
|
|
|
|
isReadOnly, hasValue := volumeLayout.readonlyVolumes[vid]
|
|
|
|
volumeLayout.accessLock.RUnlock()
|
|
|
|
|
|
|
|
if hasValue && isReadOnly {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2018-12-31 16:06:52 +08:00
|
|
|
glog.V(2).Infof("check vacuum on collection:%s volume:%d", c.Name, vid)
|
2019-12-01 23:29:41 +08:00
|
|
|
if vacuumLocationList, needVacuum := batchVacuumVolumeCheck(
|
|
|
|
grpcDialOption, volumeLayout, vid, locationList, garbageThreshold); needVacuum {
|
|
|
|
if batchVacuumVolumeCompact(grpcDialOption, volumeLayout, vid, vacuumLocationList, preallocate) {
|
|
|
|
batchVacuumVolumeCommit(grpcDialOption, volumeLayout, vid, vacuumLocationList)
|
2018-10-24 15:00:01 +08:00
|
|
|
} else {
|
2019-12-01 23:29:41 +08:00
|
|
|
batchVacuumVolumeCleanup(grpcDialOption, volumeLayout, vid, vacuumLocationList)
|
2012-11-07 17:51:43 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|