mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-11-25 11:39:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

revert changes causing wrong data

Browse Source
This commit is contained in:
Chris Lu 2020-10-24 20:07:19 -07:00
parent 9104cfa744
commit 79690b9cd8
2 changed files with 18 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
weed/filesys/dirty_page.go
View File

@ -95,8 +95,6 @@ func (pages *ContinuousDirtyPages) saveExistingLargestPageToStorage() (hasSavedD
pages.saveToStorage(maxList.ToReader(), maxList.Offset(), chunkSize)
maxList.Destroy()
return true
}

71
weed/filesys/dirty_page_interval.go
View File

@ -5,7 +5,6 @@ import (
"io"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/valyala/bytebufferpool"
)
type IntervalNode struct {
@ -13,15 +12,6 @@ type IntervalNode struct {
Offset int64
Size int64
Next *IntervalNode
Buffer *bytebufferpool.ByteBuffer
}
func (l *IntervalNode) Bytes() []byte {
data := l.Data
if data == nil {
data = l.Buffer.Bytes()
}
return data
}
type IntervalLinkedList struct {
@ -33,39 +23,16 @@ type ContinuousIntervals struct {
lists []*IntervalLinkedList
}
func NewIntervalLinkedList(head, tail *IntervalNode) *IntervalLinkedList {
list := &IntervalLinkedList{
Head: head,
Tail: tail,
}
return list
}
func (list *IntervalLinkedList) Destroy() {
for t := list.Head; t != nil; t = t.Next {
if t.Buffer != nil {
bytebufferpool.Put(t.Buffer)
}
}
}
func (list *IntervalLinkedList) Offset() int64 {
return list.Head.Offset
}
func (list *IntervalLinkedList) Size() int64 {
return list.Tail.Offset + list.Tail.Size - list.Head.Offset
}
func (list *IntervalLinkedList) addNodeToTail(node *IntervalNode) {
// glog.V(4).Infof("add to tail [%d,%d) + [%d,%d) => [%d,%d)", list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, node.Offset+node.Size, list.Head.Offset, node.Offset+node.Size)
if list.Tail.Buffer == nil {
list.Tail.Buffer = bytebufferpool.Get()
list.Tail.Buffer.Write(list.Tail.Data)
list.Tail.Data = nil
}
list.Tail.Buffer.Write(node.Data)
list.Tail.Size += int64(len(node.Data))
return
list.Tail.Next = node
list.Tail = node
}
func (list *IntervalLinkedList) addNodeToHead(node *IntervalNode) {
// glog.V(4).Infof("add to head [%d,%d) + [%d,%d) => [%d,%d)", node.Offset, node.Offset+node.Size, list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, list.Tail.Offset+list.Tail.Size)
@ -80,7 +47,7 @@ func (list *IntervalLinkedList) ReadData(buf []byte, start, stop int64) {
nodeStart, nodeStop := max(start, t.Offset), min(stop, t.Offset+t.Size)
if nodeStart < nodeStop {
// glog.V(0).Infof("copying start=%d stop=%d t=[%d,%d) t.data=%d => bufSize=%d nodeStart=%d, nodeStop=%d", start, stop, t.Offset, t.Offset+t.Size, len(t.Data), len(buf), nodeStart, nodeStop)
copy(buf[nodeStart-start:], t.Bytes()[nodeStart-t.Offset:nodeStop-t.Offset])
copy(buf[nodeStart-start:], t.Data[nodeStart-t.Offset:nodeStop-t.Offset])
}
if t.Next == nil {
@ -105,15 +72,8 @@ func subList(list *IntervalLinkedList, start, stop int64) *IntervalLinkedList {
// skip non overlapping IntervalNode
continue
}
data := t.Bytes()[nodeStart-t.Offset : nodeStop-t.Offset]
if t.Data == nil {
// need to clone if the bytes is from byte buffer
t := make([]byte, len(data))
copy(t, data)
data = t
}
nodes = append(nodes, &IntervalNode{
Data: data,
Data: t.Data[nodeStart-t.Offset : nodeStop-t.Offset],
Offset: nodeStart,
Size: nodeStop - nodeStart,
Next: nil,
@ -122,7 +82,10 @@ func subList(list *IntervalLinkedList, start, stop int64) *IntervalLinkedList {
for i := 1; i < len(nodes); i++ {
nodes[i-1].Next = nodes[i]
}
return NewIntervalLinkedList(nodes[0], nodes[len(nodes)-1])
return &IntervalLinkedList{
Head: nodes[0],
Tail: nodes[len(nodes)-1],
}
}
func (c *ContinuousIntervals) AddInterval(data []byte, offset int64) {
@ -190,7 +153,10 @@ func (c *ContinuousIntervals) AddInterval(data []byte, offset int64) {
nextList.addNodeToHead(interval)
}
if prevList == nil && nextList == nil {
c.lists = append(c.lists, NewIntervalLinkedList(interval, interval))
c.lists = append(c.lists, &IntervalLinkedList{
Head: interval,
Tail: interval,
})
}
return
@ -198,12 +164,11 @@ func (c *ContinuousIntervals) AddInterval(data []byte, offset int64) {
func (c *ContinuousIntervals) RemoveLargestIntervalLinkedList() *IntervalLinkedList {
var maxSize int64
maxIndex, maxOffset := -1, int64(-1)
maxIndex := -1
for k, list := range c.lists {
listSize := list.Size()
if maxSize < listSize || (maxSize == listSize && list.Offset() < maxOffset ) {
maxSize = listSize
maxIndex, maxOffset = k, list.Offset()
if maxSize <= list.Size() {
maxSize = list.Size()
maxIndex = k
}
}
if maxSize <= 0 {
@ -246,10 +211,10 @@ func (c *ContinuousIntervals) ReadDataAt(data []byte, startOffset int64) (maxSto
func (l *IntervalLinkedList) ToReader() io.Reader {
var readers []io.Reader
t := l.Head
readers = append(readers, util.NewBytesReader(t.Bytes()))
readers = append(readers, util.NewBytesReader(t.Data))
for t.Next != nil {
t = t.Next
readers = append(readers, bytes.NewReader(t.Bytes()))
readers = append(readers, bytes.NewReader(t.Data))
}
if len(readers) == 1 {
return readers[0]