Refactor

Author: CodeHariK

secretary/btree.go

@@ -13,18 +13,17 @@ import (
 func (s *Secretary) NewBTree(
 	collectionName string,
 	order uint8,
-	batchNumLevel uint8,
+	numLevel uint8,
 	batchBaseSize uint32,
-	batchIncrement uint8,
-	batchLength uint8,
+	increment uint8,
 	compactionBatchSize uint32,
 ) (*BTree, error) {
 	if order < MIN_ORDER || order > MAX_ORDER {
 		return nil, ErrorInvalidOrder
 	}
 
-	if batchIncrement < 110 || batchIncrement > 200 {
-		return nil, ErrorInvalidBatchIncrement
+	if increment < 110 || increment > 200 {
+		return nil, ErrorInvalidIncrement
 	}
 
 	nodeSize := uint32(order)*(KEY_SIZE+KEY_OFFSET_SIZE) + 3*POINTER_SIZE + 1
@@ -43,11 +42,10 @@ func (s *Secretary) NewBTree(
 
 		root: &Node{},
 
-		Order:          order,
-		BatchNumLevel:  batchNumLevel,
-		BatchBaseSize:  batchBaseSize,
-		BatchIncrement: batchIncrement,
-		BatchLength:    batchLength,
+		Order:         order,
+		NumLevel:      numLevel,
+		BatchBaseSize: batchBaseSize,
+		Increment:     increment,
 
 		nodeSize: uint32(nodeSize),
 
@@ -62,7 +60,7 @@ func (s *Secretary) NewBTree(
 	}
 	tree.nodePager = nodePager
 
-	recordPagers := make([]*RecordPager, batchNumLevel)
+	recordPagers := make([]*RecordPager, numLevel)
 	for i := range recordPagers {
 		pager, err := tree.NewRecordPager("record", uint8(i))
 		if err != nil {
@@ -99,36 +97,79 @@ func (tree *BTree) SaveHeader() error {
 	headerBytes = append([]byte(SECRETARY), headerBytes...)
 
 	if len(headerBytes) < SECRETARY_HEADER_LENGTH {
-		// header = append(header, make([]byte, rootHeaderSize-len(header))...)
 		headerBytes = append(headerBytes, utils.MakeByteArray(SECRETARY_HEADER_LENGTH-len(headerBytes), '-')...)
 	}
 
 	return tree.nodePager.WriteAt(headerBytes, 0)
 }
 
-func (tree *BTree) readRoot() error {
-	rootBytes, err := tree.nodePager.ReadAt(SECRETARY_HEADER_LENGTH, int32(tree.nodeSize))
+func (tree *BTree) ReadNodeAtIndex(index uint64) (*Node, error) {
+	offset := SECRETARY_HEADER_LENGTH + index*uint64(tree.nodeSize)
+
+	rootBytes, err := tree.nodePager.ReadAt(int64(offset), int32(tree.nodeSize))
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	var root Node
-	err = binstruct.Deserialize(rootBytes, &root)
+	var node Node
+	err = binstruct.Deserialize(rootBytes, &node)
+	if err != nil {
+		return nil, err
+	}
+
+	return &node, nil
+}
+
+func (tree *BTree) readRoot() error {
+	node, err := tree.ReadNodeAtIndex(0)
 	if err != nil {
 		return err
 	}
+	tree.root = node
+	return nil
+}
 
-	tree.root = &root
+func (tree *BTree) SaveNode(node *Node) error {
+	if node.Index == 0 {
+		stat, err := tree.nodePager.file.Stat()
+		if err != nil {
+			return err
+		}
+		lastFileIndex := uint64(stat.Size()) - uint64(SECRETARY_HEADER_LENGTH)/uint64(tree.nodeSize)
+		if lastFileIndex != tree.NumNodeSeq {
+			return fmt.Errorf("NumNodes dont match %d != %d", lastFileIndex, tree.NumNodeSeq)
+		}
+		tree.SaveNodeAtIndex(node, lastFileIndex)
+	} else {
+		tree.SaveNodeAtIndex(node, uint64(node.Index))
+	}
 	return nil
 }
 
-func (tree *BTree) saveRoot() error {
-	rootHeader, err := binstruct.Serialize(tree.root)
+func (tree *BTree) SaveNodeAtIndex(node *Node, index uint64) error {
+	node.Index = index
+	if node.parent != nil {
+		node.ParentIndex = node.parent.Index
+	}
+	if node.next != nil {
+		node.NextIndex = node.next.Index
+	}
+	if node.prev != nil {
+		node.PrevIndex = node.prev.Index
+	}
+
+	rootHeader, err := binstruct.Serialize(node)
 	if err != nil {
 		return err
 	}
 
-	return tree.nodePager.WriteAt(rootHeader, SECRETARY_HEADER_LENGTH)
+	offset := SECRETARY_HEADER_LENGTH + index*uint64(tree.nodeSize)
+
+	return tree.nodePager.WriteAt(rootHeader, int64(offset))
+}
+
+func (tree *BTree) saveRoot() error {
+	return tree.SaveNodeAtIndex(tree.root, 0)
 }
 
 func (s *Secretary) NewBTreeReadHeader(collectionName string) (*BTree, error) {
@@ -138,7 +179,6 @@ func (s *Secretary) NewBTreeReadHeader(collectionName string) (*BTree, error) {
 		0,
 		125,
 		0,
-		0,
 	)
 	if err != nil {
 		return nil, err
@@ -174,6 +214,20 @@ func (tree *BTree) Height() int {
 	return height
 }
 
+func (tree *BTree) Level(node *Node) int {
+	if node == nil || tree.root == nil {
+		return -1 // Return -1 for invalid nodes
+	}
+
+	level := 0
+	for node != nil && node != tree.root {
+		level++
+		node = node.parent
+	}
+
+	return level
+}
+
 func (tree *BTree) GetFirstNodePerHeight() []*Node {
 	var firstNodePerHeight []*Node
 
@@ -188,3 +242,45 @@ func (tree *BTree) GetFirstNodePerHeight() []*Node {
 
 	return firstNodePerHeight
 }
+
+func (tree *BTree) BFSCompactBatchTraversal() []*Node {
+	var compactBatch []*Node
+
+	firstNodePerHeight := tree.GetFirstNodePerHeight()
+
+	if tree.root == nil {
+		return compactBatch
+	}
+	if tree.nextCompactionNode == nil {
+		tree.nextCompactionNode = tree.root
+	}
+
+	for i := 0; i < int(tree.CompactionBatchSize); i++ {
+
+		// Ensure nextCompactionNode is not nil
+		if tree.nextCompactionNode == nil {
+			// utils.Log("tree.nextCompactionNode == nil")
+			break
+		}
+
+		compactBatch = append(compactBatch, tree.nextCompactionNode)
+
+		if tree.nextCompactionNode.next != nil {
+			tree.nextCompactionNode = tree.nextCompactionNode.next
+		} else {
+			// Ensure nextCompactionNode is not nil before calling Level()
+			level := tree.Level(tree.nextCompactionNode)
+
+			// utils.Log("level", level, "lenfirst", len(firstNodePerHeight), level > 0 && (level+1) < len(firstNodePerHeight))
+
+			if level >= 0 && (level+1) < len(firstNodePerHeight) {
+				firstNode := firstNodePerHeight[level+1]
+				tree.nextCompactionNode = firstNode
+			} else {
+				break
+			}
+		}
+	}
+
+	return compactBatch
+}

secretary/btree_test.go

@@ -46,7 +46,6 @@ func TestBtreeInvalid(t *testing.T) {
 		32,
 		1024,
 		125,
-		10,
 		1000,
 	)
 	_, invalidIncrementErr := s.NewBTree(
@@ -55,7 +54,6 @@ func TestBtreeInvalid(t *testing.T) {
 		32,
 		1024,
 		225,
-		10,
 		1000,
 	)
 	_, invalidOrderErr := s.NewBTree(
@@ -64,7 +62,6 @@ func TestBtreeInvalid(t *testing.T) {
 		32,
 		1024,
 		225,
-		10,
 		1000,
 	)
 	if invalidNameErr == nil || invalidIncrementErr == nil || invalidOrderErr == nil {

secretary/datalocation.go

@@ -4,20 +4,13 @@ import (
 	"github.com/codeharik/secretary/utils/binstruct"
 )
 
-func (datalocation DataLocation) toRecordLocation() RecordLocation {
+func ToRecordLocation(datalocation uint64) RecordLocation {
 	return RecordLocation{
 		batchLevel: uint8((uint64(datalocation) & RECORD_BATCH_LEVEL_AND) >> 56),
 		offset:     uint64(datalocation) & RECORD_BATCH_OFFSET_AND,
 	}
 }
 
-func (datalocation DataLocation) toNodeLocation() NodeLocation {
-	return NodeLocation{
-		index:       uint16((uint64(datalocation) & NODE_INDEX_AND) >> 48),
-		batchOffset: uint64(datalocation) & NODE_BATCH_OFFSET_AND,
-	}
-}
-
 func (nodes *Node) ToBytes() ([]byte, error) {
 	return binstruct.Serialize(nodes)
 }

secretary/errors.go

@@ -31,7 +31,7 @@ var (
 	ErrorRecordsNotSorted  = errors.New("Records not sorted")
 
 	ErrorInvalidOrder          = fmt.Errorf("Order must be between %d and %d", MIN_ORDER, MAX_ORDER)
-	ErrorInvalidBatchIncrement = errors.New("Batch Increment must be between 110 and 200")
+	ErrorInvalidIncrement      = errors.New("Increment must be between 110 and 200")
 	ErrorInvalidCollectionName = errors.New("Collection name is not valid, should be a-z 0-9 and with >4 & <30 characters")
 
 	// File I/O

secretary/example/main.go

@@ -21,7 +21,6 @@ func main() {
 		32,
 		1024,
 		125,
-		10,
 		1000,
 	)
 
@@ -31,7 +30,6 @@ func main() {
 		32,
 		1024*1024,
 		125,
-		10,
 		1000,
 	)
 	if userErr != nil || imagesErr != nil {

secretary/node_test.go

@@ -19,8 +19,7 @@ func dummyTree(t *testing.T, collectionName string, order uint8) (*Secretary, *B
 		32,
 		1024,
 		125,
-		10,
-		1000,
+		20,
 	)
 	if serr != nil || err != nil {
 		t.Fatal(err)
@@ -32,12 +31,12 @@ func TestSaveRoot(t *testing.T) {
 	_, tree := dummyTree(t, "TestSaveRoot", 10)
 
 	root := Node{
-		ParentOffset: 101,
-		NextOffset:   102,
-		PrevOffset:   103,
+		ParentIndex: 101,
+		NextIndex:   102,
+		PrevIndex:   103,
 
-		Keys:       [][]byte{{10, 21, 32, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
-		KeyOffsets: []DataLocation{2, 3, 4, 5, 6},
+		Keys:        [][]byte{{10, 21, 32, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
+		KeyLocation: []uint64{2, 3, 4, 5, 6},
 	}
 	tree.root = &root
 
@@ -529,14 +528,55 @@ func TestSplitInternal(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	nodes := tree.GetFirstNodePerHeight()
-	expected := []uint64{21, 7, 2, 0}
-	if len(nodes) != len(expected) {
-		t.Fatalf("Expected %d nodes, got %d", len(expected), len(nodes))
+	{
+		nodes := tree.GetFirstNodePerHeight()
+		expected := []uint64{21, 7, 2, 0}
+		if len(nodes) != len(expected) {
+			t.Fatalf("Expected %d nodes, got %d", len(expected), len(nodes))
+		}
+		for i, node := range nodes {
+			if node.NodeID != expected[i] {
+				t.Errorf("At height %d, expected NodeID %d, got %d", i, expected[i], node.NodeID)
+			}
+		}
 	}
-	for i, node := range nodes {
-		if node.NodeID != expected[i] {
-			t.Errorf("At height %d, expected NodeID %d, got %d", i, expected[i], node.NodeID)
+
+	{ // Perform batch traversal
+		compactBatch := tree.BFSCompactBatchTraversal()
+		expected := []uint64{21, 7, 20, 34, 47, 2, 6, 11, 15, 19, 25, 29, 50, 33, 38, 42, 46, 0, 1, 3}
+		if len(compactBatch) != len(expected) {
+			t.Fatalf("Expected %d nodes, got %d", len(expected), len(compactBatch))
+		}
+		for i, node := range compactBatch {
+			if node.NodeID != expected[i] {
+				t.Errorf("At index %d, expected NodeID %d, got %d", i, expected[i], node.NodeID)
+			}
+		}
+
+		compactBatch = tree.BFSCompactBatchTraversal()
+		expected = []uint64{4, 5, 8, 9, 10, 12, 13, 14, 16, 17, 18, 22, 23, 24, 26, 27, 28, 48, 49, 30}
+
+		if len(compactBatch) != len(expected) {
+			t.Fatalf("Expected %d nodes, got %d", len(expected), len(compactBatch))
+		}
+		for i, node := range compactBatch {
+			if node.NodeID != expected[i] {
+				t.Errorf("At index %d, expected NodeID %d, got %d", i, expected[i], node.NodeID)
+			}
+		}
+
+		compactBatch = tree.BFSCompactBatchTraversal()
+		expected = []uint64{31, 32, 35, 36, 37, 39, 40, 41, 43, 44, 45}
+
+		// utils.Log(utils.Map(compactBatch, func(s *Node) uint64 { return s.NodeID }))
+
+		if len(compactBatch) != len(expected) {
+			t.Fatalf("Expected %d nodes, got %d", len(expected), len(compactBatch))
+		}
+		for i, node := range compactBatch {
+			if node.NodeID != expected[i] {
+				t.Errorf("At index %d, expected NodeID %d, got %d", i, expected[i], node.NodeID)
+			}
 		}
 	}
 }