Merge pull request #1 from yourbasic/tip

Tip

Author: korthaj

README.md

@@ -4,7 +4,7 @@ Golang Bloom filter implementation
 
 ![Neutral density filter](ND-filter.jpg)
 
-*Neutral density filter, image by [Robert Emperley][re], [CC BY-SA 2.0][ccbysa].*
+*Image by [Robert Emperley][re], [CC BY-SA 2.0][ccbysa].*
 
 ### Installation
 

example_test.go

@@ -7,7 +7,7 @@ import (
 	"strconv"
 )
 
-// Create and use a Bloom filter.
+// Build a blacklist of shady websites.
 func Example_basics() {
 	// Create a Bloom filter with room for 10000 elements
 	// at a false-positives rate less than 0.5 percent.
@@ -26,27 +26,49 @@ func Example_basics() {
 	// Output: https://rascal.com seems to be shady.
 }
 
-// Count the number of false positives.
+// Estimate the number of false positives.
 func Example_falsePositives() {
 	// Create a Bloom filter with room for n elements
 	// at a false-positives rate less than 1/p.
-	n := 1000
-	p := 100
+	n, p := 10000, 100
 	filter := bloom.New(n, p)
 
 	// Add n random strings.
 	for i := 0; i < n; i++ {
-		filter.Add(strconv.FormatUint(rand.Uint64(), 10))
+		filter.Add(strconv.Itoa(rand.Int()))
 	}
 
 	// Do n random lookups and count the (mostly accidental) hits.
 	// It shouldn't be much more than n/p, and hopefully less.
 	count := 0
 	for i := 0; i < n; i++ {
-		if filter.Test(strconv.FormatUint(rand.Uint64(), 10)) {
+		if filter.Test(strconv.Itoa(rand.Int())) {
 			count++
 		}
 	}
 	fmt.Println(count, "mistakes were made.")
-	// Output: 1 mistakes were made.
+	// Output: 26 mistakes were made.
+}
+
+// Compute the union of two filters.
+func ExampleFilter_Union() {
+	// Create two Bloom filters, each with room for 1000 elements
+	// at a false-positives rate less than 1/100.
+	n, p := 1000, 100
+	f1 := bloom.New(n, p)
+	f2 := bloom.New(n, p)
+
+	// Add "0", "2", …, "498" to f1
+	for i := 0; i < n/2; i += 2 {
+		f1.Add(strconv.Itoa(i))
+	}
+
+	// Add "1", "3", …, "499" to f2
+	for i := 1; i < n/2; i += 2 {
+		f2.Add(strconv.Itoa(i))
+	}
+
+	// Compute the approximate size of f1 ∪ f2.
+	fmt.Println("f1 ∪ f2:", f1.Union(f2).Count())
+	// Output: f1 ∪ f2: 505
 }

filter.go

@@ -2,8 +2,10 @@
 //
 // Bloom filters
 //
-// A Bloom filter is a space-efficient probabilistic data structure
-// used to test set membership. A member test returns either
+// A Bloom filter is a fast and space-efficient probabilistic data structure
+// used to test set membership.
+//
+// A membership test returns either
 // ”likely member” or ”definitely not a member”. Only false positives
 // can occur: an element that has been added to the filter
 // will be identified as ”likely member”.
@@ -31,8 +33,8 @@
 //
 // This implementation is not intended for cryptographic use.
 // Each membership test makes a single call to a 128-bit MurmurHash3 function.
-// This saves on hashing without increasing the false-positives
-// probability as shown by Kirsch and Mitzenmacher.
+// This improves speed without increasing the false-positives rate
+// as shown by Kirsch and Mitzenmacher.
 //
 package bloom
 
@@ -49,23 +51,24 @@ const (
 type Filter struct {
 	data    []uint64 // Bit array, the length is a power of 2.
 	lookups int      // Lookups per query
-	count   int64    // Estimated number of unique elements
+	count   int64    // Estimate number of elements
 }
 
+// MurmurHash3 function.
 var murmur = new(digest)
 
 // New creates an empty Bloom filter with room for n elements
 // at a false-positives rate less than 1/p.
 func New(n int, p int) *Filter {
-	f := &Filter{}
 	minWords := int(0.0325 * math.Log(float64(p)) * float64(n))
 	words := 1
 	for words < minWords {
 		words *= 2
 	}
-	f.data = make([]uint64, words)
-	f.lookups = int(1.4*math.Log(float64(p)) + 1)
-	return f
+	return &Filter{
+		data:    make([]uint64, words),
+		lookups: int(1.4*math.Log(float64(p)) + 1),
+	}
 }
 
 // AddByte adds b to the filter and tells if b was already a likely member.
@@ -95,7 +98,7 @@ func (f *Filter) Add(s string) bool {
 	return f.AddByte(b)
 }
 
-// TestByte tells if b is a likely member of this filter.
+// TestByte tells if b is a likely member of the filter.
 func (f *Filter) TestByte(b []byte) bool {
 	h1, h2 := murmur.hash(b)
 	trunc := uint64(len(f.data))<<shift - 1
@@ -110,14 +113,56 @@ func (f *Filter) TestByte(b []byte) bool {
 	return true
 }
 
-// Test tells if s is a likely member of this filter.
+// Test tells if s is a likely member of the filter.
 func (f *Filter) Test(s string) bool {
 	b := make([]byte, len(s))
 	copy(b, s)
 	return f.TestByte(b)
 }
 
-// Count returns an estimate of the number of unique elements added to this filter.
+// Count returns an estimate of the number of elements in the filter.
 func (f *Filter) Count() int64 {
 	return f.count
 }
+
+// Union returns a new Bloom filter that consists of all elements
+// that belong to either f1 or f2. The two filters must be of
+// the same size n and have the same false-positives rate p.
+//
+// The resulting filter is the same as the filter created
+// from scratch using the union of the two sets.
+func (f1 *Filter) Union(f2 *Filter) *Filter {
+	if len(f1.data) != len(f2.data) || f1.lookups != f2.lookups {
+		panic("operation requires filters of the same type")
+	}
+	len := len(f1.data)
+	res := &Filter{
+		data:    make([]uint64, len),
+		lookups: f1.lookups,
+	}
+	bitCount := 0
+	for i := 0; i < len; i++ {
+		w := f1.data[i] | f2.data[i]
+		res.data[i] = w
+		bitCount += count(w)
+	}
+	// Estimate the number of elements from the bitCount.
+	m := 64 * float64(len)
+	n := m / float64(f1.lookups) * math.Log(m/(m-float64(bitCount)))
+	res.count = int64(n)
+	return res
+}
+
+// count returns the number of nonzero bits in w.
+func count(w uint64) int {
+	// Adapted from github.com/yourbasic/bit/funcs.go.
+	const maxw = 1<<64 - 1
+	const bpw = 64
+	w -= (w >> 1) & (maxw / 3)
+	w = w&(maxw/15*3) + (w>>2)&(maxw/15*3)
+	w += w >> 4
+	w &= maxw / 255 * 15
+	w *= maxw / 255
+	w >>= (bpw/8 - 1) * 8
+	return int(w)
+}

filter_test.go

@@ -48,7 +48,7 @@ func TestFilter(t *testing.T) {
 
 			member = filter.Add(s3)
 			if member {
-				t.Errorf("Add(s1) = %v; want false\n", member)
+				t.Errorf("Add(s3) = %v; want false\n", member)
 			}
 			count = filter.Count()
 			if count != 2 {
@@ -58,23 +58,52 @@ func TestFilter(t *testing.T) {
 	}
 }
 
+func TestUnion(t *testing.T) {
+	s1 := "asöldkgjaösldkgaösldkasldgjkaösldkgjöasgkdjg"
+	s2 := "elasödlnkgaölsdkfgaölsdkjfaölsdkgaölskgnaösl"
+	s3 := "aölsdgkaösldkgaösldkgjaölsdkjgaölsdkgjaösldk"
+	for n := 0; n < 100; n++ {
+		for p := 1; p <= 128; p *= 2 {
+			f1, f2 := New(n, p), New(n, p)
+			f1.Add(s1)
+			f1.Add(s2)
+			f2.Add(s2)
+			f2.Add(s3)
+			or := f1.Union(f2)
+			member := or.Test(s1)
+			if !member {
+				t.Errorf("f1.Union(f2).Test(s1) = %v; want true\n", member)
+			}
+			member = or.Test(s2)
+			if !member {
+				t.Errorf("f1.Union(f2).Test(s2) = %v; want true\n", member)
+			}
+			member = or.Test(s3)
+			if !member {
+				t.Errorf("f1.Union(f2).Test(s3) = %v; want true\n", member)
+			}
+		}
+	}
+}
+
+var fox string = "The quick brown fox jumps over the lazy dog."
+
 func BenchmarkAdd(b *testing.B) {
 	b.StopTimer()
 	filter := New(1<<30, 200)
 	b.StartTimer()
-	s := "The quick brown fox jumps over the lazy dog."
 	for i := 0; i < b.N; i++ {
-		filter.Add(s)
+		_ = filter.Add(fox)
 	}
 }
 
 func BenchmarkAddByte(b *testing.B) {
 	b.StopTimer()
 	filter := New(1<<30, 200)
 	b.StartTimer()
-	s := []byte("The quick brown fox jumps over the lazy dog.")
+	bytes := []byte(fox)
 	for i := 0; i < b.N; i++ {
-		filter.AddByte(s)
+		_ = filter.AddByte(bytes)
 	}
 }
 
@@ -83,16 +112,27 @@ func BenchmarkTest(b *testing.B) {
 	filter := New(1<<30, 200)
 	b.StartTimer()
 	for i := 0; i < b.N; i++ {
-		filter.Test("The quick brown fox jumps over the lazy dog.")
+		_ = filter.Test(fox)
 	}
 }
 
 func BenchmarkTestByte(b *testing.B) {
 	b.StopTimer()
 	filter := New(1<<30, 200)
 	b.StartTimer()
-	s := []byte("The quick brown fox jumps over the lazy dog.")
+	bytes := []byte(fox)
+	for i := 0; i < b.N; i++ {
+		_ = filter.TestByte(bytes)
+	}
+}
+
+func BenchmarkTestUnion(b *testing.B) {
+	n := 1000
+	b.StopTimer()
+	f1 := New(n, 200)
+	f2 := New(n, 200)
+	b.StartTimer()
 	for i := 0; i < b.N; i++ {
-		filter.TestByte(s)
+		_ = f1.Union(f2)
 	}
 }

hash_test.go

@@ -7,9 +7,8 @@ import (
 func TestHash(t *testing.T) {
 	d := new(digest)
 	var data = []struct {
-		h1 uint64
-		h2 uint64
-		s  string
+		h1, h2 uint64
+		s      string
 	}{
 		{0x0000000000000000, 0x0000000000000000, ""},
 		{0xcbd8a7b341bd9b02, 0x5b1e906a48ae1d19, "hello"},
@@ -23,7 +22,7 @@ func TestHash(t *testing.T) {
 			t.Errorf("hash(%q).h1 = %d; want %d\n", x.s, h1, x.h1)
 		}
 		if h2 != x.h2 {
-			t.Errorf("hash(%q).h1 = %d; want %d\n", x.s, h2, x.h2)
+			t.Errorf("hash(%q).h2 = %d; want %d\n", x.s, h2, x.h2)
 		}
 	}
 }