2019-04-19 12:43:36 +08:00
|
|
|
package needle_map
|
2017-05-27 13:51:25 +08:00
|
|
|
|
|
|
|
import (
|
2018-07-08 17:28:04 +08:00
|
|
|
. "github.com/chrislusf/seaweedfs/weed/storage/types"
|
2018-12-09 17:27:11 +08:00
|
|
|
"sort"
|
2018-07-22 08:39:10 +08:00
|
|
|
"sync"
|
2017-05-27 13:51:25 +08:00
|
|
|
)
|
|
|
|
|
2018-12-09 16:12:37 +08:00
|
|
|
const (
|
|
|
|
batch = 100000
|
|
|
|
)
|
|
|
|
|
2018-12-15 21:55:56 +08:00
|
|
|
type SectionalNeedleId uint32
|
|
|
|
|
|
|
|
const SectionalNeedleIdLimit = 1<<32 - 1
|
|
|
|
|
|
|
|
type SectionalNeedleValue struct {
|
2019-04-09 12:44:06 +08:00
|
|
|
Key SectionalNeedleId
|
|
|
|
OffsetLower OffsetLower `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
|
|
|
|
Size uint32 `comment:"Size of the data portion"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type SectionalNeedleValueExtra struct {
|
|
|
|
OffsetHigher OffsetHigher
|
2018-12-15 21:55:56 +08:00
|
|
|
}
|
|
|
|
|
2017-05-27 13:51:25 +08:00
|
|
|
type CompactSection struct {
|
|
|
|
sync.RWMutex
|
2019-04-09 12:44:06 +08:00
|
|
|
values []SectionalNeedleValue
|
|
|
|
valuesExtra []SectionalNeedleValueExtra
|
|
|
|
overflow Overflow
|
|
|
|
overflowExtra OverflowExtra
|
|
|
|
start NeedleId
|
|
|
|
end NeedleId
|
|
|
|
counter int
|
2017-05-27 13:51:25 +08:00
|
|
|
}
|
|
|
|
|
2018-12-15 21:55:56 +08:00
|
|
|
type Overflow []SectionalNeedleValue
|
2019-04-09 12:44:06 +08:00
|
|
|
type OverflowExtra []SectionalNeedleValueExtra
|
2018-12-09 17:27:11 +08:00
|
|
|
|
2018-07-08 17:28:04 +08:00
|
|
|
func NewCompactSection(start NeedleId) *CompactSection {
|
2017-05-27 13:51:25 +08:00
|
|
|
return &CompactSection{
|
2019-04-09 12:44:06 +08:00
|
|
|
values: make([]SectionalNeedleValue, batch),
|
|
|
|
valuesExtra: make([]SectionalNeedleValueExtra, batch),
|
|
|
|
overflow: Overflow(make([]SectionalNeedleValue, 0)),
|
|
|
|
overflowExtra: OverflowExtra(make([]SectionalNeedleValueExtra, 0)),
|
|
|
|
start: start,
|
2017-05-27 13:51:25 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
//return old entry size
|
2018-07-08 17:28:04 +08:00
|
|
|
func (cs *CompactSection) Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32) {
|
2017-05-27 13:51:25 +08:00
|
|
|
cs.Lock()
|
|
|
|
if key > cs.end {
|
|
|
|
cs.end = key
|
|
|
|
}
|
2018-12-15 21:55:56 +08:00
|
|
|
skey := SectionalNeedleId(key - cs.start)
|
|
|
|
if i := cs.binarySearchValues(skey); i >= 0 {
|
2019-04-09 12:44:06 +08:00
|
|
|
oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = cs.valuesExtra[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size
|
2017-05-27 13:51:25 +08:00
|
|
|
//println("key", key, "old size", ret)
|
2019-04-09 12:44:06 +08:00
|
|
|
cs.valuesExtra[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size = offset.OffsetHigher, offset.OffsetLower, size
|
2017-05-27 13:51:25 +08:00
|
|
|
} else {
|
|
|
|
needOverflow := cs.counter >= batch
|
2018-12-15 21:55:56 +08:00
|
|
|
needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > skey
|
2017-05-27 13:51:25 +08:00
|
|
|
if needOverflow {
|
|
|
|
//println("start", cs.start, "counter", cs.counter, "key", key)
|
2019-04-09 12:44:06 +08:00
|
|
|
if oldValueExtra, oldValue, found := cs.findOverflowEntry(skey); found {
|
|
|
|
oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = oldValueExtra.OffsetHigher, oldValue.OffsetLower, oldValue.Size
|
2017-05-27 13:51:25 +08:00
|
|
|
}
|
2019-04-09 12:44:06 +08:00
|
|
|
cs.setOverflowEntry(skey, offset, size)
|
2017-05-27 13:51:25 +08:00
|
|
|
} else {
|
|
|
|
p := &cs.values[cs.counter]
|
2019-04-09 12:44:06 +08:00
|
|
|
p.Key, cs.valuesExtra[cs.counter].OffsetHigher, p.OffsetLower, p.Size = skey, offset.OffsetHigher, offset.OffsetLower, size
|
2017-05-27 13:51:25 +08:00
|
|
|
//println("added index", cs.counter, "key", key, cs.values[cs.counter].Key)
|
|
|
|
cs.counter++
|
|
|
|
}
|
|
|
|
}
|
|
|
|
cs.Unlock()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2019-04-09 12:44:06 +08:00
|
|
|
func (cs *CompactSection) setOverflowEntry(skey SectionalNeedleId, offset Offset, size uint32) {
|
|
|
|
needleValue := SectionalNeedleValue{Key: skey, OffsetLower: offset.OffsetLower, Size: size}
|
|
|
|
needleValueExtra := SectionalNeedleValueExtra{OffsetHigher: OffsetHigher{}}
|
|
|
|
insertCandidate := sort.Search(len(cs.overflow), func(i int) bool {
|
|
|
|
return cs.overflow[i].Key >= needleValue.Key
|
|
|
|
})
|
|
|
|
if insertCandidate != len(cs.overflow) && cs.overflow[insertCandidate].Key == needleValue.Key {
|
|
|
|
cs.overflow[insertCandidate] = needleValue
|
|
|
|
} else {
|
|
|
|
cs.overflow = append(cs.overflow, needleValue)
|
|
|
|
cs.overflowExtra = append(cs.overflowExtra, needleValueExtra)
|
|
|
|
for i := len(cs.overflow) - 1; i > insertCandidate; i-- {
|
|
|
|
cs.overflow[i] = cs.overflow[i-1]
|
|
|
|
cs.overflowExtra[i] = cs.overflowExtra[i-1]
|
|
|
|
}
|
|
|
|
cs.overflow[insertCandidate] = needleValue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (cs *CompactSection) findOverflowEntry(key SectionalNeedleId) (nve SectionalNeedleValueExtra, nv SectionalNeedleValue, found bool) {
|
|
|
|
foundCandidate := sort.Search(len(cs.overflow), func(i int) bool {
|
|
|
|
return cs.overflow[i].Key >= key
|
|
|
|
})
|
|
|
|
if foundCandidate != len(cs.overflow) && cs.overflow[foundCandidate].Key == key {
|
|
|
|
return cs.overflowExtra[foundCandidate], cs.overflow[foundCandidate], true
|
|
|
|
}
|
|
|
|
return nve, nv, false
|
|
|
|
}
|
|
|
|
|
|
|
|
func (cs *CompactSection) deleteOverflowEntry(key SectionalNeedleId) {
|
|
|
|
length := len(cs.overflow)
|
|
|
|
deleteCandidate := sort.Search(length, func(i int) bool {
|
|
|
|
return cs.overflow[i].Key >= key
|
|
|
|
})
|
|
|
|
if deleteCandidate != length && cs.overflow[deleteCandidate].Key == key {
|
|
|
|
for i := deleteCandidate; i < length-1; i++ {
|
|
|
|
cs.overflow[i] = cs.overflow[i+1]
|
|
|
|
cs.overflowExtra[i] = cs.overflowExtra[i+1]
|
|
|
|
}
|
|
|
|
cs.overflow = cs.overflow[0 : length-1]
|
|
|
|
cs.overflowExtra = cs.overflowExtra[0 : length-1]
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-05-27 13:51:25 +08:00
|
|
|
//return old entry size
|
2018-07-08 17:28:04 +08:00
|
|
|
func (cs *CompactSection) Delete(key NeedleId) uint32 {
|
2018-12-15 21:55:56 +08:00
|
|
|
skey := SectionalNeedleId(key - cs.start)
|
2017-05-27 13:51:25 +08:00
|
|
|
cs.Lock()
|
|
|
|
ret := uint32(0)
|
2018-12-15 21:55:56 +08:00
|
|
|
if i := cs.binarySearchValues(skey); i >= 0 {
|
2019-01-26 14:05:44 +08:00
|
|
|
if cs.values[i].Size > 0 && cs.values[i].Size != TombstoneFileSize {
|
2017-05-27 13:51:25 +08:00
|
|
|
ret = cs.values[i].Size
|
2019-01-26 14:05:44 +08:00
|
|
|
cs.values[i].Size = TombstoneFileSize
|
2017-05-27 13:51:25 +08:00
|
|
|
}
|
|
|
|
}
|
2019-04-09 12:44:06 +08:00
|
|
|
if _, v, found := cs.findOverflowEntry(skey); found {
|
|
|
|
cs.deleteOverflowEntry(skey)
|
2017-05-27 13:51:25 +08:00
|
|
|
ret = v.Size
|
|
|
|
}
|
|
|
|
cs.Unlock()
|
|
|
|
return ret
|
|
|
|
}
|
2018-07-08 17:28:04 +08:00
|
|
|
func (cs *CompactSection) Get(key NeedleId) (*NeedleValue, bool) {
|
2017-05-27 13:51:25 +08:00
|
|
|
cs.RLock()
|
2018-12-15 21:55:56 +08:00
|
|
|
skey := SectionalNeedleId(key - cs.start)
|
2019-04-09 12:44:06 +08:00
|
|
|
if ve, v, ok := cs.findOverflowEntry(skey); ok {
|
2017-05-27 13:51:25 +08:00
|
|
|
cs.RUnlock()
|
2019-04-09 12:44:06 +08:00
|
|
|
nv := toNeedleValue(ve, v, cs)
|
2018-12-15 21:55:56 +08:00
|
|
|
return &nv, true
|
2017-05-27 13:51:25 +08:00
|
|
|
}
|
2018-12-15 21:55:56 +08:00
|
|
|
if i := cs.binarySearchValues(skey); i >= 0 {
|
2017-05-27 13:51:25 +08:00
|
|
|
cs.RUnlock()
|
2019-04-09 12:44:06 +08:00
|
|
|
nv := toNeedleValue(cs.valuesExtra[i], cs.values[i], cs)
|
2018-12-15 21:55:56 +08:00
|
|
|
return &nv, true
|
2017-05-27 13:51:25 +08:00
|
|
|
}
|
|
|
|
cs.RUnlock()
|
|
|
|
return nil, false
|
|
|
|
}
|
2018-12-15 21:55:56 +08:00
|
|
|
func (cs *CompactSection) binarySearchValues(key SectionalNeedleId) int {
|
2018-12-23 03:05:29 +08:00
|
|
|
x := sort.Search(cs.counter, func(i int) bool {
|
|
|
|
return cs.values[i].Key >= key
|
|
|
|
})
|
|
|
|
if x == cs.counter {
|
|
|
|
return -1
|
2017-05-27 13:51:25 +08:00
|
|
|
}
|
2018-12-23 03:05:29 +08:00
|
|
|
if cs.values[x].Key > key {
|
|
|
|
return -2
|
2017-05-27 13:51:25 +08:00
|
|
|
}
|
2018-12-23 03:05:29 +08:00
|
|
|
return x
|
2017-05-27 13:51:25 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
//This map assumes mostly inserting increasing keys
|
|
|
|
//This map assumes mostly inserting increasing keys
|
|
|
|
type CompactMap struct {
|
|
|
|
list []*CompactSection
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewCompactMap() *CompactMap {
|
|
|
|
return &CompactMap{}
|
|
|
|
}
|
|
|
|
|
2018-07-08 17:28:04 +08:00
|
|
|
func (cm *CompactMap) Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32) {
|
2017-05-27 13:51:25 +08:00
|
|
|
x := cm.binarySearchCompactSection(key)
|
2018-12-15 21:55:56 +08:00
|
|
|
if x < 0 || (key-cm.list[x].start) > SectionalNeedleIdLimit {
|
2018-12-23 03:05:29 +08:00
|
|
|
// println(x, "adding to existing", len(cm.list), "sections, starting", key)
|
2018-12-09 13:45:14 +08:00
|
|
|
cs := NewCompactSection(key)
|
|
|
|
cm.list = append(cm.list, cs)
|
2017-05-27 13:51:25 +08:00
|
|
|
x = len(cm.list) - 1
|
|
|
|
//keep compact section sorted by start
|
2018-12-23 03:05:29 +08:00
|
|
|
for x >= 0 {
|
|
|
|
if x > 0 && cm.list[x-1].start > key {
|
2018-12-09 13:45:14 +08:00
|
|
|
cm.list[x] = cm.list[x-1]
|
2018-12-23 03:05:29 +08:00
|
|
|
// println("shift", x, "start", cs.start, "to", x-1)
|
2017-05-27 13:51:25 +08:00
|
|
|
x = x - 1
|
|
|
|
} else {
|
2018-12-09 13:45:14 +08:00
|
|
|
cm.list[x] = cs
|
2018-12-23 03:05:29 +08:00
|
|
|
// println("cs", x, "start", cs.start)
|
2017-05-27 13:51:25 +08:00
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2018-12-23 03:05:29 +08:00
|
|
|
// println(key, "set to section[", x, "].start", cm.list[x].start)
|
2017-05-27 13:51:25 +08:00
|
|
|
return cm.list[x].Set(key, offset, size)
|
|
|
|
}
|
2018-07-08 17:28:04 +08:00
|
|
|
func (cm *CompactMap) Delete(key NeedleId) uint32 {
|
2017-05-27 13:51:25 +08:00
|
|
|
x := cm.binarySearchCompactSection(key)
|
|
|
|
if x < 0 {
|
|
|
|
return uint32(0)
|
|
|
|
}
|
|
|
|
return cm.list[x].Delete(key)
|
|
|
|
}
|
2018-07-08 17:28:04 +08:00
|
|
|
func (cm *CompactMap) Get(key NeedleId) (*NeedleValue, bool) {
|
2017-05-27 13:51:25 +08:00
|
|
|
x := cm.binarySearchCompactSection(key)
|
|
|
|
if x < 0 {
|
|
|
|
return nil, false
|
|
|
|
}
|
|
|
|
return cm.list[x].Get(key)
|
|
|
|
}
|
2018-07-08 17:28:04 +08:00
|
|
|
func (cm *CompactMap) binarySearchCompactSection(key NeedleId) int {
|
2018-12-23 03:52:58 +08:00
|
|
|
l, h := 0, len(cm.list)-1
|
|
|
|
if h < 0 {
|
|
|
|
return -5
|
2017-05-27 13:51:25 +08:00
|
|
|
}
|
2018-12-23 03:52:58 +08:00
|
|
|
if cm.list[h].start <= key {
|
|
|
|
if cm.list[h].counter < batch || key <= cm.list[h].end {
|
|
|
|
return h
|
|
|
|
}
|
|
|
|
return -4
|
|
|
|
}
|
|
|
|
for l <= h {
|
|
|
|
m := (l + h) / 2
|
|
|
|
if key < cm.list[m].start {
|
|
|
|
h = m - 1
|
|
|
|
} else { // cm.list[m].start <= key
|
|
|
|
if cm.list[m+1].start <= key {
|
|
|
|
l = m + 1
|
|
|
|
} else {
|
|
|
|
return m
|
|
|
|
}
|
|
|
|
}
|
2018-12-23 03:05:29 +08:00
|
|
|
}
|
2018-12-23 03:52:58 +08:00
|
|
|
return -3
|
2017-05-27 13:51:25 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Visit visits all entries or stop if any error when visiting
|
2019-05-19 13:46:24 +08:00
|
|
|
func (cm *CompactMap) AscendingVisit(visit func(NeedleValue) error) error {
|
2017-05-27 13:51:25 +08:00
|
|
|
for _, cs := range cm.list {
|
|
|
|
cs.RLock()
|
2019-05-19 13:46:24 +08:00
|
|
|
var i, j int
|
2019-05-29 12:29:07 +08:00
|
|
|
for i, j = 0, 0; i < len(cs.overflow) && j < len(cs.values) && j < cs.counter; {
|
2019-05-19 13:46:24 +08:00
|
|
|
if cs.overflow[i].Key < cs.values[j].Key {
|
|
|
|
if err := visit(toNeedleValue(cs.overflowExtra[i], cs.overflow[i], cs)); err != nil {
|
|
|
|
cs.RUnlock()
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
i++
|
2019-05-29 12:29:07 +08:00
|
|
|
} else if cs.overflow[i].Key == cs.values[j].Key {
|
2019-05-19 13:46:24 +08:00
|
|
|
j++
|
2019-05-29 12:29:07 +08:00
|
|
|
} else {
|
2019-05-19 13:46:24 +08:00
|
|
|
if err := visit(toNeedleValue(cs.valuesExtra[j], cs.values[j], cs)); err != nil {
|
|
|
|
cs.RUnlock()
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
j++
|
|
|
|
}
|
|
|
|
}
|
2019-05-29 12:29:07 +08:00
|
|
|
for ; i < len(cs.overflow); i++ {
|
2019-05-19 13:46:24 +08:00
|
|
|
if err := visit(toNeedleValue(cs.overflowExtra[i], cs.overflow[i], cs)); err != nil {
|
2017-05-27 13:51:25 +08:00
|
|
|
cs.RUnlock()
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
2019-05-29 12:29:07 +08:00
|
|
|
for ; j < len(cs.values) && j < cs.counter; j++ {
|
2019-05-19 13:46:24 +08:00
|
|
|
if err := visit(toNeedleValue(cs.valuesExtra[j], cs.values[j], cs)); err != nil {
|
|
|
|
cs.RUnlock()
|
|
|
|
return err
|
2017-05-27 13:51:25 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
cs.RUnlock()
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
2018-12-09 17:27:11 +08:00
|
|
|
|
2019-04-09 12:44:06 +08:00
|
|
|
func toNeedleValue(snve SectionalNeedleValueExtra, snv SectionalNeedleValue, cs *CompactSection) NeedleValue {
|
|
|
|
offset := Offset{
|
|
|
|
OffsetHigher: snve.OffsetHigher,
|
|
|
|
OffsetLower: snv.OffsetLower,
|
2018-12-09 17:27:11 +08:00
|
|
|
}
|
2019-04-09 12:44:06 +08:00
|
|
|
return NeedleValue{Key: NeedleId(snv.Key) + cs.start, Offset: offset, Size: snv.Size}
|
2018-12-09 17:27:11 +08:00
|
|
|
}
|
|
|
|
|
2019-04-09 12:44:06 +08:00
|
|
|
func (nv NeedleValue) toSectionalNeedleValue(cs *CompactSection) (SectionalNeedleValue, SectionalNeedleValueExtra) {
|
|
|
|
return SectionalNeedleValue{
|
2019-05-29 12:29:07 +08:00
|
|
|
SectionalNeedleId(nv.Key - cs.start),
|
|
|
|
nv.Offset.OffsetLower,
|
|
|
|
nv.Size,
|
|
|
|
}, SectionalNeedleValueExtra{
|
|
|
|
nv.Offset.OffsetHigher,
|
|
|
|
}
|
2018-12-15 21:55:56 +08:00
|
|
|
}
|