add NodeStore

weed/util/bptree/bpmap.go

@@ -9,9 +9,9 @@ import (
  */
 type BpMap BpTree
 
-func NewBpMap(node_size int) *BpMap {
+func NewBpMap(node_size int, nodeStore NodeStore) *BpMap {
 	return &BpMap{
-		root: NewLeaf(node_size),
+		root: NewLeaf(node_size, nodeStore),
 	}
 }
 
@@ -47,7 +47,7 @@ func (self *BpMap) Remove(key ItemKey) (value ItemValue, err error) {
 		return nil, err
 	}
 	if new_root == nil {
-		new_root = NewLeaf(ns)
+		new_root = NewLeaf(ns, self.root.nodeStore)
 		err = new_root.persist()
 		self.setRoot(new_root)
 	} else {

weed/util/bptree/bptree.go

@@ -12,9 +12,9 @@ type BpTree struct {
 
 type loc_iterator func() (i int, leaf *BpNode, li loc_iterator)
 
-func NewBpTree(node_size int) *BpTree {
+func NewBpTree(node_size int, nodeStore NodeStore) *BpTree {
 	return &BpTree{
-		root: NewLeaf(node_size),
+		root: NewLeaf(node_size, nodeStore),
 	}
 }
 
@@ -93,7 +93,7 @@ func (self *BpTree) RemoveWhere(key ItemKey, where WhereFunc) (err error) {
 		return err
 	}
 	if new_root == nil {
-		new_root = NewLeaf(ns)
+		new_root = NewLeaf(ns, self.root.nodeStore)
 		err = new_root.persist()
 		self.setRoot(new_root)
 	} else {

weed/util/bptree/bptree_node.go

@@ -2,13 +2,11 @@ package bptree
 
 type ItemKey Hashable
 type ItemValue Equatable
-type PersistFunc func(node *BpNode) error
-type DestroyFunc func(node *BpNode) error
 
-var (
-	PersistFn PersistFunc
-	DestroyFn DestroyFunc
-)
+type NodeStore interface {
+	PersistFunc(node *BpNode) error
+	DestroyFunc(node *BpNode) error
+}
 
 type BpNode struct {
 	keys        []ItemKey
@@ -18,9 +16,10 @@ type BpNode struct {
 	prev        *BpNode
 	protoNodeId int64
 	protoNode   *ProtoNode
+	nodeStore   NodeStore
 }
 
-func NewInternal(size int) *BpNode {
+func NewInternal(size int, nodeStore NodeStore) *BpNode {
 	if size < 0 {
 		panic(NegativeSize())
 	}
@@ -28,10 +27,11 @@ func NewInternal(size int) *BpNode {
 		keys:        make([]ItemKey, 0, size),
 		pointers:    make([]*BpNode, 0, size),
 		protoNodeId: GetProtoNodeId(),
+		nodeStore:   nodeStore,
 	}
 }
 
-func NewLeaf(size int) *BpNode {
+func NewLeaf(size int, nodeStore NodeStore) *BpNode {
 	if size < 0 {
 		panic(NegativeSize())
 	}
@@ -39,6 +39,7 @@ func NewLeaf(size int) *BpNode {
 		keys:        make([]ItemKey, 0, size),
 		values:      make([]ItemValue, 0, size),
 		protoNodeId: GetProtoNodeId(),
+		nodeStore:   nodeStore,
 	}
 }
 
@@ -187,7 +188,7 @@ func (self *BpNode) put(key ItemKey, value ItemValue) (root *BpNode, err error)
 		return a, nil
 	}
 	// else we have root split
-	root = NewInternal(self.Capacity())
+	root = NewInternal(self.Capacity(), self.nodeStore)
 	root.put_kp(a.keys[0], a)
 	root.put_kp(b.keys[0], b)
 	return root, root.persist()
@@ -256,7 +257,7 @@ func (self *BpNode) internal_split(key ItemKey, ptr *BpNode) (a, b *BpNode, err
 		return nil, nil, BpTreeError("Tried to split an internal block on duplicate key")
 	}
 	a = self
-	b = NewInternal(self.Capacity())
+	b = NewInternal(self.Capacity(), self.nodeStore)
 	balance_nodes(a, b, key)
 	if b.Len() > 0 && key.Less(b.keys[0]) {
 		if err := a.put_kp(key, ptr); err != nil {
@@ -310,7 +311,7 @@ func (self *BpNode) leaf_split(key ItemKey, value ItemValue) (a, b *BpNode, err
 		return self.pure_leaf_split(key, value)
 	}
 	a = self
-	b = NewLeaf(self.Capacity())
+	b = NewLeaf(self.Capacity(), self.nodeStore)
 	insert_linked_list_node(b, a, a.getNext())
 	balance_nodes(a, b, key)
 	if b.Len() > 0 && key.Less(b.keys[0]) {
@@ -342,7 +343,7 @@ func (self *BpNode) pure_leaf_split(key ItemKey, value ItemValue) (a, b *BpNode,
 		return nil, nil, BpTreeError("Expected a pure leaf node")
 	}
 	if key.Less(self.keys[0]) {
-		a = NewLeaf(self.Capacity())
+		a = NewLeaf(self.Capacity(), self.nodeStore)
 		b = self
 		if err := a.put_kv(key, value); err != nil {
 			return nil, nil, err
@@ -358,7 +359,7 @@ func (self *BpNode) pure_leaf_split(key ItemKey, value ItemValue) (a, b *BpNode,
 			}
 			return a, nil, a.persist()
 		} else {
-			b = NewLeaf(self.Capacity())
+			b = NewLeaf(self.Capacity(), self.nodeStore)
 			if err := b.put_kv(key, value); err != nil {
 				return nil, nil, err
 			}

weed/util/bptree/bptree_store_test.go

@@ -5,29 +5,38 @@ import (
 	"testing"
 )
 
+type nodeStorePrintlnImpl struct {
+}
+
+func (n *nodeStorePrintlnImpl) PersistFunc(node *BpNode) error {
+	println("saving node", node.protoNodeId)
+	return nil
+}
+func (n *nodeStorePrintlnImpl) DestroyFunc(node *BpNode) error {
+	println("delete node", node.protoNodeId)
+	return nil
+}
+
 func TestAddRemove(t *testing.T) {
-	tree := NewBpTree(5)
-	PersistFn = func(node *BpNode) error {
-		println("saving", node.protoNodeId)
-		return nil
-	}
-	DestroyFn = func(node *BpNode) error {
-		println("delete", node.protoNodeId)
-		return nil
-	}
-	for i:=0;i<32;i++{
+
+	tree := NewBpTree(3, &nodeStorePrintlnImpl{})
+	for i:=0;i<9;i++{
 		println("++++++++++", i)
 		tree.Add(String(fmt.Sprintf("%02d", i)), nil)
 		printTree(tree.root, "")
 	}
 
-	if !tree.Has(String("30")) {
+	if !tree.Has(String("08")) {
 		t.Errorf("lookup error")
 	}
-	tree.RemoveWhere(String("30"), func(value ItemValue) bool {
-		return true
-	})
-	if tree.Has(String("30")) {
+	for i:=5;i<9;i++{
+		println("----------", i)
+		tree.RemoveWhere(String(fmt.Sprintf("%02d", i)), func(value ItemValue) bool {
+			return true
+		})
+		printTree(tree.root, "")
+	}
+	if tree.Has(String("08")) {
 		t.Errorf("remove error")
 	}
 }

weed/util/bptree/bptree_test.go

@@ -79,7 +79,7 @@ func BenchmarkBpTree(b *testing.B) {
 
 	b.StartTimer()
 	for i := 0; i < b.N; i++ {
-		t := NewBpTree(23)
+		t := NewBpTree(23, nil)
 		for _, r := range recs {
 			t.Add(r.key, r.value)
 		}
@@ -207,7 +207,7 @@ func TestAddHasCountFindIterateRemove(t *testing.T) {
 		}
 	}
 	for i := 2; i < 64; i++ {
-		test(NewBpTree(i))
+		test(NewBpTree(i, nil))
 	}
 }
 
@@ -271,11 +271,11 @@ func TestBpMap(t *testing.T) {
 		}
 	}
 
-	test(NewBpMap(23))
+	test(NewBpMap(23, nil))
 }
 
 func Test_get_start(t *testing.T) {
-	root := NewLeaf(2)
+	root := NewLeaf(2, nil)
 	root, err := root.put(Int(1), Int(1))
 	if err != nil {
 		t.Error(err)
@@ -344,7 +344,7 @@ func Test_get_start(t *testing.T) {
 }
 
 func Test_get_end(t *testing.T) {
-	root := NewLeaf(3)
+	root := NewLeaf(3, nil)
 	root, err := root.put(Int(1), Int(1))
 	if err != nil {
 		t.Fatal(err)
@@ -388,7 +388,7 @@ func Test_get_end(t *testing.T) {
 }
 
 func Test_put_no_root_split(t *testing.T) {
-	a := NewLeaf(2)
+	a := NewLeaf(2, nil)
 	if err := a.put_kv(Int(1), Int(1)); err != nil {
 		t.Error(err)
 	}
@@ -423,7 +423,7 @@ func Test_put_no_root_split(t *testing.T) {
 }
 
 func Test_put_root_split(t *testing.T) {
-	a := NewLeaf(2)
+	a := NewLeaf(2, nil)
 	p, err := a.put(Int(1), Int(1))
 	if err != nil {
 		t.Error(err)
@@ -472,8 +472,8 @@ func Test_put_root_split(t *testing.T) {
 }
 
 func Test_internal_insert_no_split(t *testing.T) {
-	a := NewInternal(3)
-	leaf := NewLeaf(1)
+	a := NewInternal(3, nil)
+	leaf := NewLeaf(1, nil)
 	if err := leaf.put_kv(Int(1), Int(1)); err != nil {
 		t.Error(err)
 	}
@@ -500,8 +500,8 @@ func Test_internal_insert_no_split(t *testing.T) {
 }
 
 func Test_internal_insert_split_less(t *testing.T) {
-	a := NewInternal(3)
-	leaf := NewLeaf(1)
+	a := NewInternal(3, nil)
+	leaf := NewLeaf(1, nil)
 	if err := leaf.put_kv(Int(1), Int(1)); err != nil {
 		t.Error(err)
 	}
@@ -534,7 +534,7 @@ func Test_internal_insert_split_less(t *testing.T) {
 }
 
 func Test_internal_split_less(t *testing.T) {
-	a := NewInternal(3)
+	a := NewInternal(3, nil)
 	if err := a.put_kp(Int(1), nil); err != nil {
 		t.Error(err)
 	}
@@ -564,7 +564,7 @@ func Test_internal_split_less(t *testing.T) {
 }
 
 func Test_internal_split_equal(t *testing.T) {
-	a := NewInternal(3)
+	a := NewInternal(3, nil)
 	if err := a.put_kp(Int(1), nil); err != nil {
 		t.Error(err)
 	}
@@ -581,7 +581,7 @@ func Test_internal_split_equal(t *testing.T) {
 }
 
 func Test_internal_split_greater(t *testing.T) {
-	a := NewInternal(3)
+	a := NewInternal(3, nil)
 	if err := a.put_kp(Int(1), nil); err != nil {
 		t.Error(err)
 	}
@@ -611,7 +611,7 @@ func Test_internal_split_greater(t *testing.T) {
 }
 
 func Test_leaf_insert_no_split(t *testing.T) {
-	a := NewLeaf(3)
+	a := NewLeaf(3, nil)
 	insert_linked_list_node(a, nil, nil)
 	if err := a.put_kv(Int(1), Int(1)); err != nil {
 		t.Error(err)
@@ -637,7 +637,7 @@ func Test_leaf_insert_no_split(t *testing.T) {
 
 // tests the defer to split logic
 func Test_leaf_insert_split_less(t *testing.T) {
-	a := NewLeaf(3)
+	a := NewLeaf(3, nil)
 	insert_linked_list_node(a, nil, nil)
 	if err := a.put_kv(Int(1), Int(1)); err != nil {
 		t.Error(err)
@@ -668,7 +668,7 @@ func Test_leaf_insert_split_less(t *testing.T) {
 }
 
 func Test_leaf_split_less(t *testing.T) {
-	a := NewLeaf(3)
+	a := NewLeaf(3, nil)
 	insert_linked_list_node(a, nil, nil)
 	if err := a.put_kv(Int(1), Int(1)); err != nil {
 		t.Error(err)
@@ -699,7 +699,7 @@ func Test_leaf_split_less(t *testing.T) {
 }
 
 func Test_leaf_split_equal(t *testing.T) {
-	a := NewLeaf(3)
+	a := NewLeaf(3, nil)
 	insert_linked_list_node(a, nil, nil)
 	if err := a.put_kv(Int(1), Int(1)); err != nil {
 		t.Error(err)
@@ -730,7 +730,7 @@ func Test_leaf_split_equal(t *testing.T) {
 }
 
 func Test_leaf_split_greater(t *testing.T) {
-	a := NewLeaf(3)
+	a := NewLeaf(3, nil)
 	insert_linked_list_node(a, nil, nil)
 	if err := a.put_kv(Int(1), Int(1)); err != nil {
 		t.Error(err)
@@ -762,13 +762,13 @@ func Test_leaf_split_greater(t *testing.T) {
 
 // tests the defer logic
 func Test_pure_leaf_insert_split_less(t *testing.T) {
-	a := NewLeaf(2)
+	a := NewLeaf(2, nil)
 	insert_linked_list_node(a, nil, nil)
-	b := NewLeaf(2)
+	b := NewLeaf(2, nil)
 	insert_linked_list_node(b, a, nil)
-	c := NewLeaf(2)
+	c := NewLeaf(2, nil)
 	insert_linked_list_node(c, b, nil)
-	d := NewLeaf(2)
+	d := NewLeaf(2, nil)
 	insert_linked_list_node(d, c, nil)
 	if err := a.put_kv(Int(3), Int(1)); err != nil {
 		t.Error(err)
@@ -835,13 +835,13 @@ func Test_pure_leaf_insert_split_less(t *testing.T) {
 }
 
 func Test_pure_leaf_split_less(t *testing.T) {
-	a := NewLeaf(2)
+	a := NewLeaf(2, nil)
 	insert_linked_list_node(a, nil, nil)
-	b := NewLeaf(2)
+	b := NewLeaf(2, nil)
 	insert_linked_list_node(b, a, nil)
-	c := NewLeaf(2)
+	c := NewLeaf(2, nil)
 	insert_linked_list_node(c, b, nil)
-	d := NewLeaf(2)
+	d := NewLeaf(2, nil)
 	insert_linked_list_node(d, c, nil)
 	if err := a.put_kv(Int(3), Int(1)); err != nil {
 		t.Error(err)
@@ -908,13 +908,13 @@ func Test_pure_leaf_split_less(t *testing.T) {
 }
 
 func Test_pure_leaf_split_equal(t *testing.T) {
-	a := NewLeaf(2)
+	a := NewLeaf(2, nil)
 	insert_linked_list_node(a, nil, nil)
-	b := NewLeaf(2)
+	b := NewLeaf(2, nil)
 	insert_linked_list_node(b, a, nil)
-	c := NewLeaf(2)
+	c := NewLeaf(2, nil)
 	insert_linked_list_node(c, b, nil)
-	d := NewLeaf(2)
+	d := NewLeaf(2, nil)
 	insert_linked_list_node(d, c, nil)
 	if err := a.put_kv(Int(3), Int(1)); err != nil {
 		t.Error(err)
@@ -972,13 +972,13 @@ func Test_pure_leaf_split_equal(t *testing.T) {
 }
 
 func Test_pure_leaf_split_greater(t *testing.T) {
-	a := NewLeaf(2)
+	a := NewLeaf(2, nil)
 	insert_linked_list_node(a, nil, nil)
-	b := NewLeaf(2)
+	b := NewLeaf(2, nil)
 	insert_linked_list_node(b, a, nil)
-	c := NewLeaf(2)
+	c := NewLeaf(2, nil)
 	insert_linked_list_node(c, b, nil)
-	d := NewLeaf(2)
+	d := NewLeaf(2, nil)
 	insert_linked_list_node(d, c, nil)
 	if err := a.put_kv(Int(3), Int(1)); err != nil {
 		t.Error(err)
@@ -1042,13 +1042,13 @@ func Test_pure_leaf_split_greater(t *testing.T) {
 }
 
 func Test_find_end_of_pure_run(t *testing.T) {
-	a := NewLeaf(2)
+	a := NewLeaf(2, nil)
 	insert_linked_list_node(a, nil, nil)
-	b := NewLeaf(2)
+	b := NewLeaf(2, nil)
 	insert_linked_list_node(b, a, nil)
-	c := NewLeaf(2)
+	c := NewLeaf(2, nil)
 	insert_linked_list_node(c, b, nil)
-	d := NewLeaf(2)
+	d := NewLeaf(2, nil)
 	insert_linked_list_node(d, c, nil)
 	if err := a.put_kv(Int(3), Int(1)); err != nil {
 		t.Error(err)
@@ -1078,13 +1078,13 @@ func Test_find_end_of_pure_run(t *testing.T) {
 }
 
 func Test_insert_linked_list_node(t *testing.T) {
-	a := NewLeaf(1)
+	a := NewLeaf(1, nil)
 	insert_linked_list_node(a, nil, nil)
-	b := NewLeaf(2)
+	b := NewLeaf(2, nil)
 	insert_linked_list_node(b, a, nil)
-	c := NewLeaf(3)
+	c := NewLeaf(3, nil)
 	insert_linked_list_node(c, b, nil)
-	d := NewLeaf(4)
+	d := NewLeaf(4, nil)
 	insert_linked_list_node(d, a, b)
 	if a.getPrev() != nil {
 		t.Errorf("expected a.prev == nil")
@@ -1113,13 +1113,13 @@ func Test_insert_linked_list_node(t *testing.T) {
 }
 
 func Test_remove_linked_list_node(t *testing.T) {
-	a := NewLeaf(1)
+	a := NewLeaf(1, nil)
 	insert_linked_list_node(a, nil, nil)
-	b := NewLeaf(2)
+	b := NewLeaf(2, nil)
 	insert_linked_list_node(b, a, nil)
-	c := NewLeaf(3)
+	c := NewLeaf(3, nil)
 	insert_linked_list_node(c, b, nil)
-	d := NewLeaf(4)
+	d := NewLeaf(4, nil)
 	insert_linked_list_node(d, a, b)
 	if a.getPrev() != nil {
 		t.Errorf("expected a.prev == nil")
@@ -1188,8 +1188,8 @@ func Test_remove_linked_list_node(t *testing.T) {
 }
 
 func Test_balance_leaf_nodes_with_dup(t *testing.T) {
-	a := NewLeaf(3)
-	b := NewLeaf(3)
+	a := NewLeaf(3, nil)
+	b := NewLeaf(3, nil)
 	if err := a.put_kv(Int(1), Int(1)); err != nil {
 		t.Error(err)
 	}
@@ -1209,8 +1209,8 @@ func Test_balance_leaf_nodes_with_dup(t *testing.T) {
 }
 
 func Test_balance_leaf_nodes(t *testing.T) {
-	a := NewLeaf(7)
-	b := NewLeaf(7)
+	a := NewLeaf(7, nil)
+	b := NewLeaf(7, nil)
 	if err := a.put_kv(Int(1), Int(1)); err != nil {
 		t.Error(err)
 	}
@@ -1258,8 +1258,8 @@ func Test_balance_leaf_nodes(t *testing.T) {
 }
 
 func Test_balance_internal_nodes(t *testing.T) {
-	a := NewInternal(6)
-	b := NewInternal(6)
+	a := NewInternal(6, nil)
+	b := NewInternal(6, nil)
 	if err := a.put_kp(Int(1), nil); err != nil {
 		t.Error(err)
 	}

weed/util/bptree/getter_setter.go

@@ -45,14 +45,14 @@ func (self *BpNode) maybePersist(shouldPersist bool) error {
 	return self.persist()
 }
 func (self *BpNode) persist() error {
-	if PersistFn != nil {
-		return PersistFn(self)
+	if self.nodeStore != nil {
+		return self.nodeStore.PersistFunc(self)
 	}
 	return nil
 }
 func (self *BpNode) destroy() error {
-	if DestroyFn != nil {
-		return DestroyFn(self)
+	if self.nodeStore != nil {
+		return self.nodeStore.DestroyFunc(self)
 	}
 	return nil
 }

weed/util/bptree/serde_test.go

@@ -0,0 +1,46 @@
+package bptree
+
+import (
+	"fmt"
+	"testing"
+)
+
+type nodeStoreMapImpl struct {
+	m map[int64]*ProtoNode
+}
+
+func (n *nodeStoreMapImpl) PersistFunc(node *BpNode) error {
+	println("saving node", node.protoNodeId)
+	n.m[node.protoNodeId] = node.protoNode
+	return nil
+}
+func (n *nodeStoreMapImpl) DestroyFunc(node *BpNode) error {
+	println("delete node", node.protoNodeId)
+	delete(n.m, node.protoNodeId)
+	return nil
+}
+
+func TestSerDe(t *testing.T) {
+
+	nodeStore := &nodeStoreMapImpl{
+		m: make(map[int64]*ProtoNode),
+	}
+
+	tree := NewBpTree(3, nodeStore)
+
+	for i:=0;i<32;i++{
+		println("add", i)
+		tree.Add(String(fmt.Sprintf("%02d", i)), nil)
+	}
+
+	for i:=5;i<9;i++{
+		println("----------", i)
+		tree.RemoveWhere(String(fmt.Sprintf("%02d", i)), func(value ItemValue) bool {
+			return true
+		})
+		printTree(tree.root, "")
+	}
+
+
+
+}