Data structure

Author: CodeHariK

Readme.md

@@ -9,6 +9,8 @@
 * [Build a NoSQL Database From Scratch in 1000 Lines of Code](https://medium.com/better-programming/build-a-nosql-database-from-the-scratch-in-1000-lines-of-code-8ed1c15ed924)
 * [Writing a SQL database from scratch in Go: 1. SELECT, INSERT, CREATE and a REPL](https://notes.eatonphil.com/database-basics.html)
 
+* [Database Engine Development](https://www.youtube.com/playlist?list=PLm7R-cUo29CXVu9a9TzBEwSQ9JPVGmISg)
+
 * https://github.com/cmu-db/bustub
 
 ##
@@ -47,14 +49,60 @@
 
 * [An introduction to Conflict-Free Replicated Data Types (CRDTs)](https://www.youtube.com/watch?v=gZP2VUmH05A)
 
+## Data Structure
+
+* [Heaps, heapsort, and priority queues - Inside code](https://www.youtube.com/watch?v=pLIajuc31qk)
+* [Trie data structure - Inside code](https://www.youtube.com/watch?v=qA8l8TAMyig)
+* [Compressed trie](https://www.youtube.com/watch?v=qakGXuOW1S8)
+
 ## Probablistic Data structures
 
+* [Hello Interview : Bloom Filters, Count-Min Sketch, HyperLogLog](https://www.youtube.com/watch?v=IgyU0iFIoqM)
+
+* [Probablistic data structure lectures](https://www.youtube.com/playlist?list=PL2mpR0RYFQsAR5RyB54FyEE9vUiGtCSZM)
+
+### Bloom filter 
+
+* [Wikipedia](https://en.wikipedia.org/wiki/Bloom_filter)
+* [mCoding : Bloom Filters](https://www.youtube.com/watch?v=qZNJTh2NEiU)
+* [Number0 : Bloom Filters](https://www.youtube.com/watch?v=eCUm4U3WDpM)
+* [ByteByteGo : Bloom Filters](https://www.youtube.com/watch?v=V3pzxngeLqw)
+* [Spanning Tree : What Are Bloom Filters?](https://www.youtube.com/watch?v=kfFacplFY4Y)
+* [ByteMonk : Bloom Filters](https://www.youtube.com/watch?v=GT0En1dGntY)
+
+Bloom filter is a space-efficient probabilistic data structure, that is used to test whether an element is a member of a set. False positive matches are possible, but false negatives are not - in other words, a query returns either "possibly in set" or "definitely not in set". Elements can be added to the set, but not removed.
+
+A Bloom filter is a representation of a set of _n_ items, where the main requirement is to make membership queries; _i.e._, whether an item is a member of a set.
+
+#### Uses
+##### Cache filtering
+Content delivery networks deploy web caches around the world to cache and serve web content to users with greater performance and reliability. A key application of Bloom filters is their use in efficiently determining which web objects to store in these web caches. To prevent caching one-hit-wonders, a Bloom filter is used to keep track of all URLs that are accessed by users.
+##### Web Crawler
+
+### HyperLogLog
+
 * [PapersWeLove : HyperLogLog](https://www.youtube.com/watch?v=y3fTaxA8PkU)
-* [A problem so hard even Google relies on Random Chance](https://www.youtube.com/watch?v=lJYufx0bfpw)
 * [The Algorithm with the Best Name - HyperLogLog Explained](https://www.youtube.com/watch?v=2PlrMCiUN_s)
+* [A problem so hard even Google relies on Random Chance](https://www.youtube.com/watch?v=lJYufx0bfpw)
+* [Counting BILLIONS with Just Kilobytes](https://www.youtube.com/watch?v=f69hh3KgFEk)
+* https://github.com/tylertreat/BoomFilters/blob/master/hyperloglog.go
 
+### Count–min sketch
+
+* [Wikepedia](https://en.wikipedia.org/wiki/Count%E2%80%93min_sketch)
+* https://github.com/tylertreat/BoomFilters/blob/master/countmin.go
 * [Count-min Sketch](https://www.youtube.com/watch?v=Okdjn7o4q8E)
 
+The goal of the basic version of the count–min sketch is to consume a stream of events, one at a time, and count the frequency of the different types of events in the stream.
+
+### HeavyKeeper TopK
+
+* [Understanding Probabilistic Data Structures](https://www.youtube.com/watch?v=2Dzc7fxA0us)
+
+### T Digest
+
+* [Sketching Data with T Digest](https://www.youtube.com/watch?v=ETUYhEZRtWE)
+
 ## Cache
 
 * [TinyLFU: A Highly Efficient Cache Admission Policy](https://dgraph.io/blog/refs/TinyLFU%20-%20A%20Highly%20Efficient%20Cache%20Admission%20Policy.pdf)

datastructure/bloomfilter.go

@@ -0,0 +1,77 @@
+package datastructure
+
+import (
+	"errors"
+
+	"github.com/zeebo/xxh3"
+)
+
+type BloomFilter struct {
+	bitset []byte
+	hasher []*xxh3.Hasher
+	m      uint64
+	k      uint64
+}
+
+func (b *BloomFilter) setBit(pos uint64) {
+	b.bitset[pos/8] |= 1 << (pos % 8)
+}
+
+func (b *BloomFilter) getBit(pos uint64) bool {
+	return b.bitset[pos/8]&(1<<(pos%8)) != 0
+}
+
+func (b *BloomFilter) Add(data []byte) {
+	for i := 0; i < int(b.k); i++ {
+		h := b.hasher[i]
+		h.Reset()
+		h.Write(data)
+		hash := h.Sum64()
+		b.setBit(hash % b.m)
+	}
+}
+
+func (b *BloomFilter) Check(data []byte) bool {
+	for i := 0; i < int(b.k); i++ {
+		h := b.hasher[i]
+		h.Reset()
+		h.Write(data)
+		hash := h.Sum64()
+		if !b.getBit(hash % b.m) {
+			return false
+		}
+	}
+	return true
+}
+
+func (b *BloomFilter) Union(a *BloomFilter) (err error) {
+	if b.m != a.m {
+		return errors.New("the bloom filters have the different sizes")
+	}
+
+	if b.k != a.k {
+		return errors.New("the bloom filters have the different number of hash functions")
+	}
+
+	for i := uint64(0); i < b.m; i++ {
+		if b.getBit(i) || a.getBit(i) {
+			b.setBit(i)
+		}
+	}
+
+	return nil
+}
+
+func NewBloomFilter(m uint64, k uint64) BloomFilter {
+	hasher := make([]*xxh3.Hasher, k)
+	for i := range hasher {
+		h := xxh3.NewSeed(uint64(i))
+		hasher[i] = h
+	}
+	return BloomFilter{
+		bitset: make([]byte, (m+7)/8),
+		hasher: hasher,
+		m:      m,
+		k:      k,
+	}
+}

datastructure/bloomfilter_test.go

@@ -0,0 +1,67 @@
+package datastructure
+
+import (
+	"testing"
+)
+
+func TestBloomFilter(t *testing.T) {
+	bf := NewBloomFilter(1024, 3) // 1024 bits, 3 hash functions
+
+	wordsToAdd := [][]byte{
+		[]byte("hello"),
+		[]byte("world"),
+		[]byte("golang"),
+	}
+
+	wordsNotAdded := [][]byte{
+		[]byte("python"),
+		[]byte("java"),
+		[]byte("rust"),
+	}
+
+	// Add words
+	for _, word := range wordsToAdd {
+		bf.Add(word)
+	}
+
+	// Check words that were added
+	for _, word := range wordsToAdd {
+		if !bf.Check(word) {
+			t.Errorf("Expected word %q to be found in BloomFilter", word)
+		}
+	}
+
+	// Check words that were NOT added
+	falsePositiveCount := 0
+	for _, word := range wordsNotAdded {
+		if bf.Check(word) {
+			falsePositiveCount++
+		}
+	}
+
+	// Allow a small false positive rate
+	if falsePositiveCount > 1 {
+		t.Errorf("Too many false positives: got %d, want <= 1", falsePositiveCount)
+	}
+}
+
+func BenchmarkBloomFilterAdd(b *testing.B) {
+	bf := NewBloomFilter(1<<20, 5) // 1 million bits, 5 hash functions
+	data := []byte("benchmark-data")
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bf.Add(data)
+	}
+}
+
+func BenchmarkBloomFilterCheck(b *testing.B) {
+	bf := NewBloomFilter(1<<20, 5)
+	data := []byte("benchmark-data")
+	bf.Add(data)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bf.Check(data)
+	}
+}

datastructure/countminsketch.go

@@ -0,0 +1,76 @@
+package datastructure
+
+import (
+	"errors"
+
+	"github.com/zeebo/xxh3"
+)
+
+type CountMinSketch struct {
+	freq   [][]uint64
+	hasher []*xxh3.Hasher
+	m      uint64
+	k      uint64
+	count  uint64
+}
+
+func NewCountMinSketch(m, k uint64) CountMinSketch {
+	hasher := make([]*xxh3.Hasher, k)
+	for i := range hasher {
+		h := xxh3.NewSeed(uint64(i))
+		hasher[i] = h
+	}
+
+	freq := make([][]uint64, k)
+	for i := uint64(0); i < k; i++ {
+		freq[i] = make([]uint64, m) // each row is size m
+	}
+
+	return CountMinSketch{
+		freq:   freq,
+		hasher: hasher,
+		m:      m,
+		k:      k,
+	}
+}
+
+func (c *CountMinSketch) Add(data []byte) {
+	for i := uint64(0); i < c.k; i++ {
+		pos := xxh3.HashSeed(data, i) % c.m
+		c.freq[i][pos]++
+	}
+	c.count++
+}
+
+func (c *CountMinSketch) Count(data []byte) uint64 {
+	min := uint64(^uint64(0)) // max uint64
+	for i := uint64(0); i < c.k; i++ {
+		pos := xxh3.HashSeed(data, i) % c.m
+		count := c.freq[i][pos]
+		if count < min {
+			min = count
+		}
+	}
+	return min
+}
+
+// Merge combines this CountMinSketch with another. Returns an error if the
+// matrix width and depth are not equal.
+func (c *CountMinSketch) Merge(other *CountMinSketch) error {
+	if c.k != other.k {
+		return errors.New("matrix depth must match")
+	}
+
+	if c.m != other.m {
+		return errors.New("matrix width must match")
+	}
+
+	for i := uint64(0); i < c.k; i++ {
+		for j := uint64(0); j < c.m; j++ {
+			c.freq[i][j] += other.freq[i][j]
+		}
+	}
+
+	c.count += other.count
+	return nil
+}

datastructure/go.mod

@@ -0,0 +1,8 @@
+module github.com/codeharik/datastructure
+
+go 1.24.0
+
+require (
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
+)

datastructure/go.sum

@@ -0,0 +1,4 @@
+github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=