BasicReconcile

Author: magicmilo

hybridestimator.go

@@ -46,7 +46,6 @@ func (h *HybridEstimator) BuildSignature(keys [][]byte) {
 }
 
 func (h *HybridEstimator) EstimateSizeDifference(remote *HybridEstimator) int {
-
 	count := 0
 
 	for level := h.Depth - 1; level >= -1; level-- {
@@ -55,9 +54,7 @@ func (h *HybridEstimator) EstimateSizeDifference(remote *HybridEstimator) int {
 		} else if level < 2 { //MinHash
 			mh := h.MinHashset[level]
 			fmt.Println(mh.Difference(remote.MinHashset[level]))
-
 		} else { //IBF Strata
-
 			ibf := h.IBFset[level]
 			remotelevel := remote.IBFset[level]
 			ibf.Subtract(remotelevel)

ibf.go

@@ -85,6 +85,31 @@ func (f *IBF) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
+//Takes and sets values in the IBF directly
+func (f *IBF) SetIBF(data IBFSerialization) error {
+	bitset, err := hex.DecodeString(data.Data)
+	if err != nil {
+		return err
+	}
+
+	f.Size = data.Size
+	f.Keysize = data.Keysize
+	f.Hashset = data.Hashset
+	f.Countset = data.Countset
+	f.Bitset = bitset
+
+	return nil
+}
+
+func (f *IBF) GetIBF() IBFSerialization {
+	return IBFSerialization{
+		f.Size,
+		f.Keysize,
+		f.Hashset,
+		f.Countset,
+		hex.EncodeToString(f.Bitset)}
+}
+
 // Hashes returns an array of hash values resulting from the specified `key`.
 // This implementation uses the 128-bit x86 murmur3 hash and returns the
 // following, in order:

minhash.go

@@ -1,7 +1,6 @@
 package reconcile
 
 import (
-	"fmt"
 	"math"
 )
 
@@ -18,69 +17,66 @@ import (
 //if h(row) < M(i, c)
 //M(i,c) = h(row)
 
-func MinHashDifference(sigA [][]uint64, sigB [][]uint64) int {
-	if len(sigA) != len(sigB) {
-		fmt.Println("Hash count doesn't match")
-		return 0 //should throw error
-	}
-	if len(sigA[0]) != len(sigB[0]) {
-		fmt.Println("Keysizes don't match")
-		return 0 //should throw error
-	}
-
-	diff := 0
-	bitwidth := len(sigA[0])
-	for row := range sigA {
-		for col := 0; col < bitwidth; col++ {
-			if sigA[row][col] == sigB[row][col] {
-				if sigA[row][col] != math.MaxUint64 && sigB[row][col] != math.MaxUint64 {
-					diff++
-				}
-			}
-		}
-	}
-
-	return diff
+type MinHash struct {
+	keysize   int
+	signature [][]uint64
+	hashcount uint32
+	keycount  int
 }
 
-//Should probably return a custom struct
-func GetMinHashSignature(keys [][]byte, hashcount uint32) [][]uint64 {
-	//dont precompute permutation hashset
-	//build signature table of hashcount rows
-	//hashseed is signature row
-	keysize := len(keys[0])
+func NewMinHash(hashcount uint32, keysize int) *MinHash {
+	var hashseed uint32
 
-	mh := make([][]uint64, hashcount)
-	for row := range mh {
-		mh[row] = make([]uint64, keysize*8)
+	//Initialise signature with maximum values
+	signature := make([][]uint64, hashcount)
+	for row := range signature {
+		signature[row] = make([]uint64, keysize*8)
 	}
 
-	var hashseed uint32
-
-	//init signature with values (maximum)
 	for hashseed = 0; hashseed < hashcount; hashseed++ { //0 to hashcount
 		for bitindex := 0; bitindex < (keysize * 8); bitindex++ { //for each bit in key
-			mh[hashseed][bitindex] = math.MaxUint64
+			signature[hashseed][bitindex] = math.MaxUint64
 		} //scan through bits
 	} //for each hash
+	return &MinHash{keysize, signature, hashcount, 0}
+}
+
+func (mh *MinHash) Add(key []byte) {
+	var hashseed uint32
 
-	for _, key := range keys {
-		for hashseed = 0; hashseed < hashcount; hashseed++ { //0 to hashcount
-			sum := Sum128x32(key, hashseed)
-			hash := uint64(sum[0]) % uint64(len(keys))
-			for byteindex, keybyte := range key { //for each byte of key
-				var pattern uint8 = 1
-				for bitindex := 0; bitindex < 8; bitindex++ { //for each bit in key
-					if keybyte&pattern != 0 { //do if found bit & not already found
-						if hash < mh[hashseed][(byteindex*8)+bitindex] {
-							mh[hashseed][(byteindex*8)+bitindex] = hash
-						} //if hash is less than then update
-					} //if bit is 1
-					pattern <<= 1
-				} //scan through bits
-			} //scan through bytes of key
-		} //for each hash
-	} //for each key
+	for hashseed = 0; hashseed < mh.hashcount; hashseed++ { //0 to hashcount
+		sum := Sum128x32(key, hashseed)
+		hash := uint64(sum[0]) % uint64(mh.hashcount)
+		for byteindex, keybyte := range key { //for each byte of key
+			var pattern uint8 = 1
+			for bitindex := 0; bitindex < 8; bitindex++ { //for each bit in key
+				if keybyte&pattern != 0 { //do if found bit & not already found
+					if hash < mh.signature[hashseed][(byteindex*8)+bitindex] {
+						mh.signature[hashseed][(byteindex*8)+bitindex] = hash
+					} //if hash is less than then update
+				} //if bit is 1
+				pattern <<= 1
+			} //scan through bits
+		} //scan through bytes of key
+	} //for each hash
+	mh.keycount++
+}
 
-	return mh
-} //GetSignature
+//Returns Jacquard similiarity score
+func (mh *MinHash) Difference(remote *MinHash) int {
+	//Catch errors
+
+	match := 0
+	bitwidth := mh.keysize * 8
+	for row := range mh.signature {
+		for col := 0; col < bitwidth; col++ {
+			if mh.signature[row][col] == remote.signature[row][col] {
+				if mh.signature[row][col] != math.MaxUint64 && remote.signature[row][col] != math.MaxUint64 {
+					match++
+				}
+			} //if they match
+		} //for each bit
+	} //for each hash/row in signature
+	score := float64(match) / float64(bitwidth*int(mh.hashcount))
+	return int(((1.0 - score) / (1.0 + score)) * float64(mh.keycount+remote.keycount))
+}

minhash_test.go

@@ -7,13 +7,12 @@ import (
 )
 
 func TestMinHash(t *testing.T) {
-	numElements := 1000
-	numDifferences := 1000
+	numElements := 100
+	numDifferences := 80
 	var hashCount uint32 = 100
 	keysize := 32
-
-	localkeys := [][]byte{}
-	remotekeys := [][]byte{}
+	local := NewMinHash(hashCount, keysize)
+	remote := NewMinHash(hashCount, keysize)
 
 	for i := 0; i < numElements; i++ {
 		element := make([]byte, keysize)
@@ -22,8 +21,8 @@ func TestMinHash(t *testing.T) {
 			t.Error("Could not get random bytes for set element")
 			return
 		}
-		localkeys = append(localkeys, element)
-		remotekeys = append(remotekeys, element)
+		local.Add(element)
+		remote.Add(element)
 	}
 
 	for i := 0; i < numDifferences; i++ {
@@ -34,20 +33,16 @@ func TestMinHash(t *testing.T) {
 			return
 		}
 		// Add to a set at random
-		diffSet := &localkeys
+		diffSet := local
 		if rand.Intn(2) == 0 {
-			diffSet = &remotekeys
+			diffSet = remote
 		}
-		*diffSet = append(*diffSet, element)
+		diffSet.Add(element)
 	}
 
-	sigA := GetMinHashSignature(localkeys, hashCount)
-	sigB := GetMinHashSignature(remotekeys, hashCount)
-
-	diff := MinHashDifference(sigA, sigB)
+	diff := local.Difference(remote)
 
-	fmt.Printf("MinHash Diff %v out of %v\n", diff, int(hashCount)*keysize*8)
-	//fmt.Printf("%v%%\n", diff, int(hashCount)*keysize*8)
+	fmt.Printf("MinHash Diff %v vs actual: %v\n", diff, numDifferences)
 
 	fmt.Printf("End MinHash \n")
 }

reconcile.go

@@ -1,46 +1,68 @@
 package reconcile
 
-//Inputs:
-//-A local set
-//-A remote strata estimator
-//-A remote MiHash signature
-
-//Estimates the difference between the local and remote set
-//Builds an IBF of size estimated * factor
-//Returns two sets of what is missing from either data sets
-
-//Usage steps:
-//NewSetReconciler(set)
-//Estimate size of difference
-//Get Missing Elements
-
-//Size of difference involves partitioning keys into n levels
-//where size of leveln = len(set) / (2^n)
-//Large Partitions are handed to minhash estimator
-//Level 1 - 2
-//Strata estimator operates on 3..
-
-//1: Both parties
-//  Generate Two Signatures
-//  level1-2 minhash
-//  further strata
-//filter := NewIBF(cells, keysize)
-
-//create the reconciler
-//exchange set sizes and agree on minimum size
-//TEMPORARY!!
-//use local set to build minhash and strata estimator data structure
-//exchange with other party
-//estimate difference size
-//build ibf
-//exchange ibfs
-//get difference
+import (
+	"math"
+)
+
+//Create reconciler with local keys knowing the remote set size
+//Generate strata signature and transmit
+//Receive remote strata signature, estimate size
+//Build IBF of 'size' and transmit signature
+//Receive remote IBF signature and calculate difference
 
 type Reconcile struct {
+	Keyset    [][]byte
+	Estimator *Strata
+	Depth     int
+	DiffSize  int //default 0
+}
+
+//Creates a set reconciler and populates a size estimator with all local keys
+func NewReconcile(keys [][]byte, remotesetsize int) *Reconcile {
+	//Ugly
+	var depth int
+	if remotesetsize > len(keys) {
+		depth = int(math.Ceil(math.Log2(float64(remotesetsize))))
+	} else {
+		depth = int(math.Ceil(math.Log2(float64(len(keys)))))
+	}
+
+	estimator := NewStrata(80, len(keys[0]), depth)
+	estimator.Populate(keys)
+
+	return &Reconcile{keys, estimator, depth, 0}
+}
+
+func (r *Reconcile) GetDifferenceSizeEstimator() ([]byte, error) {
+	return r.Estimator.MarshalStrataJSON()
+}
+
+//Takes JSON estimator data from remote and estimates size of difference
+func (r *Reconcile) EstimateDifferenceSize(data []byte) error {
+	remote := NewStrata(80, len(r.Keyset[0]), r.Depth)
+	remote.UnmarshalStrataJSON(data)
+	r.DiffSize = r.Estimator.Estimate(remote)
+	return nil
+}
 
+//Generates signature of ibf dataset
+//Must be called after estimating difference size
+func (r *Reconcile) GetIBFSignature(size int) ([]byte, error) {
+	ibf := NewIBF(size, len(r.Keyset[0]))
+	for _, key := range r.Keyset {
+		ibf.Add(key)
+	}
+	return ibf.MarshalJSON()
 }
 
-func NewSetReconciler(keys [][]byte) {
+func (r *Reconcile) GetDifference(size int, remotesignature []byte) (a [][]byte, b [][]byte, ok bool) {
+	ibf := NewIBF(size, len(r.Keyset[0]))
+	for _, key := range r.Keyset {
+		ibf.Add(key)
+	}
+	remoteibf := NewIBF(size, len(r.Keyset[0]))
+	remoteibf.UnmarshalJSON(remotesignature)
+	ibf.Subtract(remoteibf)
+	return ibf.Decode()
 
-    IBFset := make([]*IBF, )
 }