Skip to content

Commit 201ff08

Browse files
CodeHariKCodeHariK
committed
File
1 parent d7602fe commit 201ff08

12 files changed

Lines changed: 456 additions & 95 deletions

File tree

‎secretary/btree.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66

77
"github.com/codeharik/secretary/utils"
88
"github.com/codeharik/secretary/utils/binstruct"
9+
"github.com/codeharik/secretary/utils/file"
910
)
1011

1112
func (s *Secretary) NewBTree(
@@ -31,7 +32,7 @@ func (s *Secretary) NewBTree(
3132
return nil, ErrorInvalidCollectionName
3233
}
3334

34-
if err := utils.EnsureDir(fmt.Sprintf("%s/%s", SECRETARY, safeCollectionName)); err != nil {
35+
if err := file.EnsureDir(fmt.Sprintf("%s/%s", SECRETARY, safeCollectionName)); err != nil {
3536
return nil, err
3637
}
3738

‎secretary/secretary.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"os"
55

66
"github.com/codeharik/secretary/utils"
7+
"github.com/codeharik/secretary/utils/file"
78
)
89

910
func New() (*Secretary, error) {
@@ -15,7 +16,7 @@ func New() (*Secretary, error) {
1516

1617
dirPath := "./SECRETARY"
1718

18-
err := utils.EnsureDir(dirPath)
19+
err := file.EnsureDir(dirPath)
1920
if err != nil {
2021
return nil, err
2122
}

‎secretary/utils/binstruct/binary_struct.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import (
1616

1717
// Serialize struct to binary []byte (Little-Endian)
1818
// bin : type name
19-
// byte : number of bytes used for length of string or []byte
19+
// lenbyte : number of bytes used for length of string or []byte
2020
// max : max length of string or []byte
2121
// array_elem_len : max length (array elements) in (array of (array elements)), [][]byte [][]int32 [][]float64
2222
func Serialize(s interface{}) ([]byte, error) {
@@ -558,12 +558,12 @@ func getSizeFromField(field reflect.StructField) int {
558558

559559
// Get the bit from struct tag (default to 8 if not specified)
560560
func getByteFromField(field reflect.StructField) int {
561-
tag := field.Tag.Get("byte")
561+
tag := field.Tag.Get("lenbyte")
562562

563563
var byte int
564564
_, err := fmt.Sscanf(tag, "%d", &byte)
565-
if err != nil && byte != 2 && byte != 3 && byte != 4 {
566-
return 1
565+
if err != nil || byte < 1 || byte > 8 {
566+
return 4
567567
}
568568
return byte
569569
}

‎secretary/utils/binstruct/binary_struct_test.go‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package binstruct
22

33
import (
44
"bytes"
5+
"encoding/gob"
6+
"fmt"
57
"testing"
68

79
"github.com/codeharik/secretary/utils"
@@ -20,12 +22,12 @@ type testStruct struct {
2022
Ffloat64 float64 `bin:"Ffloat64"`
2123
Fstring string `bin:"Fstring"`
2224

23-
Fstring_4_30 string `bin:"Fstring_4_30" byte:"4" max:"30"`
25+
Fstring_4_30 string `bin:"Fstring_4_30" lenbyte:"1" max:"30"`
2426
Fstring_30 string `bin:"Fstring_30" max:"30"`
2527
Fstring_10 string `bin:"Fstring_10" max:"10"`
2628

2729
Fbytes []byte `bin:"Fbytes"`
28-
Fbytes_300 []byte `bin:"Fbytes_300" max:"300"`
30+
Fbytes_300 []byte `bin:"Fbytes_300" lenbyte:"1" max:"300"`
2931

3032
Fstring_Empty string `bin:"FstringEmpty"`
3133
Fbytes_Empty []byte `bin:"FbytesEmpty"`
@@ -80,7 +82,7 @@ func TestBinaryStructSerialize(t *testing.T) {
8082
"Array": {
8183
true,
8284
testStruct{
83-
Fint64_array: []int64{125, 2000},
85+
Fint64_array: utils.GenerateRandomSlice[int64](2000),
8486
Fint32_array_20: utils.GenerateRandomSlice[int32](20),
8587
},
8688
},
@@ -127,6 +129,16 @@ func TestBinaryStructSerialize(t *testing.T) {
127129
t.Fatal(err)
128130
}
129131

132+
{
133+
var buf bytes.Buffer
134+
enc := gob.NewEncoder(&buf)
135+
err := enc.Encode(test.s)
136+
if err != nil {
137+
t.Fatal(err)
138+
}
139+
fmt.Println("gob", len(buf.Bytes()), "bin", len(binaryData), " bin/gob", 100*len(binaryData)/len(buf.Bytes()))
140+
}
141+
130142
var d testStruct
131143
err = Deserialize(binaryData, &d)
132144
if err != nil {
@@ -143,11 +155,10 @@ func TestBinaryStructSerialize(t *testing.T) {
143155
t.Fatal(err)
144156
}
145157

146-
// utils.Log("binaryData", (binaryData), "s", test.s, "d", d, "jsonH", string(jsonH), "jsonD", string(jsonD))
147158
if test.equal != (bytes.Compare(jsonH, jsonD) == 0) {
148159
utils.Log("nCompare Should be equal", test.equal,
149-
len(jsonH), string(jsonH),
150-
len(jsonD), string(jsonD),
160+
"jsonH", len(jsonH), string(jsonH), "",
161+
"jsonD", len(jsonD), string(jsonD), "",
151162
)
152163
t.Fatal()
153164
}

‎secretary/utils/file.go‎

Lines changed: 0 additions & 17 deletions
This file was deleted.

‎secretary/utils/file/file.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package file
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"path/filepath"
7+
)
8+
9+
// Ensure directory exists
10+
func EnsureDir(path string) error {
11+
if _, err := os.Stat(path); os.IsNotExist(err) {
12+
err := os.MkdirAll(path, 0o755) // Create directory and parents if needed
13+
if err != nil {
14+
return fmt.Errorf("%s %v", path, err)
15+
}
16+
}
17+
return nil
18+
}
19+
20+
// EnsureFile ensures a file exists, creating parent directories if needed.
21+
func EnsureFile(path string) error {
22+
dir := filepath.Dir(path)
23+
24+
// Ensure parent directory exists
25+
if err := EnsureDir(dir); err != nil {
26+
return err
27+
}
28+
29+
// Check if file exists
30+
if _, err := os.Stat(path); os.IsNotExist(err) {
31+
// Create empty file
32+
file, err := os.Create(path)
33+
if err != nil {
34+
return fmt.Errorf("failed to create file %s: %v", path, err)
35+
}
36+
file.Close()
37+
}
38+
39+
return nil
40+
}

‎secretary/utils/file/filechunk.go‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
package file
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"hash/crc32"
7+
"io"
8+
"os"
9+
"path/filepath"
10+
)
11+
12+
const (
13+
ChunkSize = 1024 // Adjust chunk size
14+
)
15+
16+
// Metadata structure
17+
type Metadata struct {
18+
Filename string `json:"filename"`
19+
FileSize int64 `json:"file_size"`
20+
NumChunks int32 `json:"num_chunks"`
21+
Chunks []string `json:"chunks"` // "chunkname:index_hash"
22+
}
23+
24+
func splitFile(filePath string, metadataFile string) error {
25+
file, err := os.Open(filePath)
26+
if err != nil {
27+
return err
28+
}
29+
defer file.Close()
30+
31+
// Get file size
32+
stat, err := file.Stat()
33+
if err != nil {
34+
return err
35+
}
36+
fileSize := stat.Size()
37+
38+
metadir := filepath.Dir(metadataFile)
39+
40+
// Ensure chunk directory exists
41+
os.MkdirAll(metadir, os.ModePerm)
42+
43+
metadata := Metadata{
44+
Filename: filePath,
45+
FileSize: fileSize,
46+
NumChunks: int32(fileSize/ChunkSize) + 1,
47+
Chunks: make([]string, int32(fileSize/ChunkSize)+1),
48+
}
49+
50+
buffer := make([]byte, ChunkSize)
51+
index := 0
52+
53+
for {
54+
n, err := file.Read(buffer)
55+
if n > 0 {
56+
hash := crc32.ChecksumIEEE(buffer[:n])
57+
58+
// Store metadata with format: "chunkname:index_hash"
59+
metadata.Chunks[index] = fmt.Sprintf("%d_%08x", index, hash)
60+
61+
// Save chunk
62+
if err := os.WriteFile(filepath.Join(metadir, metadata.Chunks[index]), buffer[:n], 0o644); err != nil {
63+
return err
64+
}
65+
66+
index++
67+
}
68+
if err == io.EOF {
69+
break
70+
} else if err != nil {
71+
return err
72+
}
73+
}
74+
75+
// Save metadata
76+
metaDataBytes, _ := json.MarshalIndent(metadata, "", " ")
77+
return os.WriteFile(metadataFile, metaDataBytes, 0o644)
78+
}
79+
80+
func mergeChunks(metadataFile string, reconstructedFile string) error {
81+
// Load metadata
82+
metaDataBytes, err := os.ReadFile(metadataFile)
83+
if err != nil {
84+
return err
85+
}
86+
87+
var metadata Metadata
88+
if err := json.Unmarshal(metaDataBytes, &metadata); err != nil {
89+
return err
90+
}
91+
92+
// Create output file
93+
outFile, err := os.Create(reconstructedFile)
94+
if err != nil {
95+
return err
96+
}
97+
defer outFile.Close()
98+
99+
metadir := filepath.Dir(metadataFile)
100+
101+
// Reassemble file from chunks
102+
for _, chunkName := range metadata.Chunks {
103+
data, err := os.ReadFile(filepath.Join(metadir, chunkName))
104+
if err != nil {
105+
return err
106+
}
107+
108+
// Verify integrity
109+
hash := crc32.ChecksumIEEE(data)
110+
expectedHash := chunkName[len(chunkName)-8:] // Extract last 8 chars (hash)
111+
if fmt.Sprintf("%08x", hash) != expectedHash {
112+
return fmt.Errorf("hash mismatch for chunk: %s", expectedHash)
113+
}
114+
115+
// Append to final file
116+
_, err = outFile.Write(data)
117+
if err != nil {
118+
return err
119+
}
120+
}
121+
122+
// Verify final file size
123+
finalStat, err := os.Stat(reconstructedFile)
124+
if err != nil {
125+
return err
126+
}
127+
if finalStat.Size() != metadata.FileSize {
128+
return fmt.Errorf("file size mismatch! Expected: %d, Got: %d", metadata.FileSize, finalStat.Size())
129+
}
130+
131+
// // Delete chunks after successful merge
132+
// for _, chunkName := range metadata.Chunks {
133+
// os.Remove(filepath.Join(metadir, chunkName))
134+
// }
135+
// os.Remove(metadataFile)
136+
137+
return nil
138+
}

0 commit comments

Comments
 (0)