Skip to content

Commit 444efdb

Browse files
CodeHariKCodeHariK
committed
Pager
1 parent cbc9ead commit 444efdb

9 files changed

Lines changed: 67 additions & 47 deletions

File tree

‎secretary/btree.go‎

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ func (s *Secretary) NewBTree(
1414
collectionName string,
1515
order uint8,
1616
numLevel uint8,
17-
batchBaseSize uint32,
17+
baseSize uint32,
1818
increment uint8,
1919
compactionBatchSize uint32,
2020
) (*BTree, error) {
@@ -42,10 +42,10 @@ func (s *Secretary) NewBTree(
4242

4343
root: &Node{},
4444

45-
Order: order,
46-
NumLevel: numLevel,
47-
BatchBaseSize: batchBaseSize,
48-
Increment: increment,
45+
Order: order,
46+
NumLevel: numLevel,
47+
BaseSize: baseSize,
48+
Increment: increment,
4949

5050
nodeSize: uint32(nodeSize),
5151

@@ -104,20 +104,24 @@ func (tree *BTree) SaveHeader() error {
104104
}
105105

106106
func (tree *BTree) ReadNodeAtIndex(index uint64) (*Node, error) {
107-
offset := SECRETARY_HEADER_LENGTH + index*uint64(tree.nodeSize)
107+
// offset := SECRETARY_HEADER_LENGTH + index*uint64(tree.nodeSize)
108108

109-
rootBytes, err := tree.nodePager.ReadAt(int64(offset), int32(tree.nodeSize))
110-
if err != nil {
111-
return nil, err
112-
}
109+
// rootBytes, err := tree.nodePager.ReadAt(int64(offset), int32(tree.nodeSize))
110+
// if err != nil {
111+
// return nil, err
112+
// }
113113

114-
var node Node
115-
err = binstruct.Deserialize(rootBytes, &node)
116-
if err != nil {
117-
return nil, err
118-
}
114+
// var node Node
115+
// err = binstruct.Deserialize(rootBytes, &node)
116+
// if err != nil {
117+
// return nil, err
118+
// }
119+
120+
// return &node, nil
121+
122+
page, err := tree.nodePager.ReadPage(int64(index))
119123

120-
return &node, nil
124+
return page.Data, err
121125
}
122126

123127
func (tree *BTree) readRoot() error {
@@ -131,15 +135,14 @@ func (tree *BTree) readRoot() error {
131135

132136
func (tree *BTree) SaveNode(node *Node) error {
133137
if node.Index == 0 {
134-
stat, err := tree.nodePager.file.Stat()
138+
lastFileIndex, err := tree.nodePager.NumPages()
135139
if err != nil {
136140
return err
137141
}
138-
lastFileIndex := uint64(stat.Size()) - uint64(SECRETARY_HEADER_LENGTH)/uint64(tree.nodeSize)
139-
if lastFileIndex != tree.NumNodeSeq {
142+
if uint64(lastFileIndex) != tree.NumNodeSeq {
140143
return fmt.Errorf("NumNodes dont match %d != %d", lastFileIndex, tree.NumNodeSeq)
141144
}
142-
tree.SaveNodeAtIndex(node, lastFileIndex)
145+
tree.SaveNodeAtIndex(node, uint64(lastFileIndex))
143146
} else {
144147
tree.SaveNodeAtIndex(node, uint64(node.Index))
145148
}

‎secretary/datalocation.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,20 @@ func (nodes *Node) ToBytes() ([]byte, error) {
1515
return binstruct.Serialize(nodes)
1616
}
1717

18+
func (nodes *Node) NewPage(index int64) *Page[*Node] {
19+
return &Page[*Node]{
20+
Index: index,
21+
Data: &Node{},
22+
}
23+
}
24+
25+
func (records *Record) NewPage(index int64) *Page[*Record] {
26+
return &Page[*Record]{
27+
Index: index,
28+
Data: &Record{},
29+
}
30+
}
31+
1832
func (nodes *Node) FromBytes(data []byte) error {
1933
err := binstruct.Deserialize(data, nodes)
2034
if err != nil {

‎secretary/docs/secretary.excalidraw.svg‎

Lines changed: 2 additions & 2 deletions
Loading

‎secretary/pager.go‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,8 @@ func (tree *BTree) NewRecordPager(fileType string, level uint8) (*RecordPager, e
5858
}
5959

6060
// Opens or creates a file and sets up the Pager
61-
func NewPager[T PageBox](tree *BTree, fileType string, level uint8) (*Pager[T], error) {
62-
pageSize := int64(float64(tree.BatchBaseSize) * math.Pow(float64(tree.Increment)/100, float64(level)))
61+
func NewPager[T PageItem[T]](tree *BTree, fileType string, level uint8) (*Pager[T], error) {
62+
pageSize := int64(float64(tree.BaseSize) * math.Pow(float64(tree.Increment)/100, float64(level)))
6363

6464
var headerSize int64 = 0
6565

@@ -112,7 +112,7 @@ func NewPager[T PageBox](tree *BTree, fileType string, level uint8) (*Pager[T],
112112
cache, err := ristretto.NewCache(
113113
&ristretto.Config[int64, *Page[T]]{
114114
NumCounters: 10000, // Track frequency of ~10,000 items
115-
MaxCost: 1 << 20, // 1MB total cache size
115+
MaxCost: 1 << 24, // 16MB total cache size
116116
BufferItems: 64, // Batch writes for performance
117117
OnEvict: func(item *ristretto.Item[*Page[T]]) {
118118
delete(pager.dirtyPages, item.Value.Index) // Mark page as clean
@@ -154,7 +154,7 @@ func (store *Pager[T]) AllocatePage(index int64) error {
154154
return nil
155155
}
156156

157-
func (store *Pager[T]) NumPages(index int32) (int64, error) {
157+
func (store *Pager[T]) NumPages() (int64, error) {
158158
store.mu.Lock()
159159
defer store.mu.Unlock()
160160

@@ -214,8 +214,7 @@ func (store *Pager[T]) WriteAt(data []byte, offset int64) error {
214214
}
215215
}
216216

217-
{
218-
// Write data at the given offset
217+
{ // Write data at the given offset
219218
n, err := store.file.WriteAt(data, offset)
220219
if err != nil || (len(data)) != int(n) {
221220
return ErrorWritingDataAtOffset(offset, err)
@@ -247,19 +246,22 @@ func (store *Pager[T]) ReadPage(index int64) (*Page[T], error) {
247246
return cachedPage, nil
248247
}
249248

250-
page := &Page[T]{
251-
Index: index,
252-
}
249+
var item T
250+
page := item.NewPage(index) // Calls the NewPage method of PageItem[T]
251+
252+
// page := &Page[T]{
253+
// Index: index,
254+
// }
253255
// Store in Ristretto cache
254-
store.cache.Set(index, page, store.itemSize) // Cost = size of page
255-
store.cache.Wait() // Ensure writes are processed
256+
store.cache.Set(index, page, store.itemSize)
257+
store.cache.Wait()
256258

257259
store.mu.Unlock()
258260

259261
page.mu.Lock()
260262
defer page.mu.Unlock()
261263

262-
data, err := store.ReadAt(index*store.itemSize, int32(store.itemSize))
264+
data, err := store.ReadAt(index*store.itemSize+store.headerSize, int32(store.itemSize))
263265
if err != nil {
264266
return nil, err
265267
}

‎secretary/secretaryui/index.html‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@
161161
<input type="text" placeholder="Collection Name" id="collectionName" required>
162162
<input type="number" placeholder="Order" id="order" min="3" max="200" required>
163163
<input type="number" placeholder="Num Level" id="NumLevel" min="32" required>
164-
<input type="number" placeholder="Batch Base Size" id="batchBaseSize" min="1024" required>
164+
<input type="number" placeholder="Batch Base Size" id="BaseSize" min="1024" required>
165165
<input type="number" placeholder="Batch Increment" id="Increment" min="110" max="200" required>
166166
<button type="submit">Create Tree</button>
167167
</form>

‎secretary/secretaryui/src/collection.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ async function newTreeRequest(event: SubmitEvent) {
203203
CollectionName: (document.getElementById("collectionName") as HTMLInputElement).value,
204204
Order: Number((document.getElementById("order") as HTMLInputElement).value),
205205
NumLevel: Number((document.getElementById("NumLevel") as HTMLInputElement).value),
206-
BatchBaseSize: Number((document.getElementById("batchBaseSize") as HTMLInputElement).value),
206+
BaseSize: Number((document.getElementById("BaseSize") as HTMLInputElement).value),
207207
Increment: Number((document.getElementById("Increment") as HTMLInputElement).value),
208208
};
209209

‎secretary/server.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ type NewTreeRequest struct {
8484
CollectionName string `json:"CollectionName"`
8585
Order uint8 `json:"Order"`
8686
NumLevel uint8 `json:"NumLevel"`
87-
BatchBaseSize uint32 `json:"BatchBaseSize"`
87+
BaseSize uint32 `json:"BaseSize"`
8888
Increment uint8 `json:"Increment"`
8989
CompactionBatchSize uint32 `json:"compactionBatchSize"`
9090
}
@@ -100,7 +100,7 @@ func (s *Secretary) newTreeHandler(w http.ResponseWriter, r *http.Request) {
100100
req.CollectionName,
101101
req.Order,
102102
req.NumLevel,
103-
req.BatchBaseSize,
103+
req.BaseSize,
104104
req.Increment,
105105
req.CompactionBatchSize,
106106
)

‎secretary/server_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ func TestNewTreeHandler(t *testing.T) {
114114
CollectionName: "hello",
115115
Order: 10,
116116
NumLevel: 32,
117-
BatchBaseSize: 1024,
117+
BaseSize: 1024,
118118
Increment: 130,
119119
CompactionBatchSize: 1000,
120120
},
@@ -125,7 +125,7 @@ func TestNewTreeHandler(t *testing.T) {
125125
CollectionName: "hello",
126126
Order: 2,
127127
NumLevel: 32,
128-
BatchBaseSize: 104,
128+
BaseSize: 104,
129129
Increment: 105,
130130
CompactionBatchSize: 1000,
131131
},

‎secretary/types.go‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ type Secretary struct {
5151
SECRETARY (9 bytes) 9
5252
order (uint8) 10
5353
NumLevel (uint8) 11
54-
batchBaseSize (uint32) 15
54+
baseSize (uint32) 15
5555
increment (uint8) 16
5656
nodeSeq (uint64) 24
5757
numNodeSeq (uint64) 32
@@ -79,10 +79,10 @@ type BTree struct {
7979
root *Node // Root node of the tree
8080
nextCompactionNode *Node // Compaction Node For Current Batch
8181

82-
Order uint8 `json:"order" bin:"order"` // Max = 255, Order of the tree (maximum number of children)
83-
NumLevel uint8 `json:"numLevel" bin:"numLevel"` // 32, Max 256 levels
84-
BatchBaseSize uint32 `json:"batchBaseSize" bin:"batchBaseSize"` // 1024MB
85-
Increment uint8 `json:"increment" bin:"increment"` // 125 => 1.25
82+
Order uint8 `json:"order" bin:"order"` // Max = 255, Order of the tree (maximum number of children)
83+
NumLevel uint8 `json:"numLevel" bin:"numLevel"` // 32, Max 256 levels
84+
BaseSize uint32 `json:"baseSize" bin:"baseSize"` // 1024Bytes
85+
Increment uint8 `json:"increment" bin:"increment"` // 125 => 1.25
8686

8787
NodeSeq uint64 `json:"nodeSeq" bin:"nodeSeq"` // Incrementing Node sequence
8888
NumNodeSeq uint64 `json:"numNodeSeq" bin:"numNodeSeq"`
@@ -127,7 +127,8 @@ type Node struct {
127127
Keys [][]byte `bin:"Keys" array_elem_len:"16"` // (16 bytes)
128128
}
129129

130-
type PageBox interface {
130+
type PageItem[T any] interface {
131+
NewPage(index int64) *Page[T]
131132
ToBytes() ([]byte, error)
132133
FromBytes([]byte) error
133134
}
@@ -141,7 +142,7 @@ type Page[T any] struct {
141142
}
142143

143144
// Pager manages reading and writing pages.
144-
type Pager[T PageBox] struct {
145+
type Pager[T PageItem[T]] struct {
145146
file *os.File
146147

147148
level uint8 // (1.25 ^ 0)MB (1.25 ^ 1)MB ... (1.25 ^ 31)MB

0 commit comments

Comments
 (0)