Hook RAPPOR up to Metis

Author: Audrey-Randall

proxy.go

@@ -33,6 +33,11 @@ type Website struct {
 	Domain string `json:"domain,omitempty"`
 }
 
+type SvrMsg struct {
+	Cohort int `json:"cohort,omitempty"`
+	Reports []int `json:"reports,omitempty"`
+}
+
 var client = &http.Client{
 	Transport: &http.Transport{
 		Dial: (&net.Dialer{
@@ -51,6 +56,7 @@ var client = &http.Client{
 
 var transport string
 var hmacSecret string
+var cohort int
 
 /*
 Domains Metis is reasonably certain are censored are stored here.
@@ -70,7 +76,7 @@ func contains(slice []string, s string) bool {
 }
 
 func isBlocked(url *url.URL) (bool) {
-		return contains(blockedDomains, url.Hostname()) || contains(tempBlockedDomains, url.Hostname())
+	return contains(blockedDomains, url.Hostname()) || contains(tempBlockedDomains, url.Hostname())
 }
 
 func remove(s []string, e string) []string {
@@ -155,14 +161,15 @@ func updateMasterList() error {
 	s.init(p)
 	var e Encoder
 
-	e.init(p, 1, hmacSecret, &s)
+	e.init(p, cohort, hmacSecret, &s)
 	var rappor []int
 	for i := 0; i < len(blockedDomains); i++ {
-		fmt.Println("Blocked domains: ", blockedDomains)
-		fmt.Println(e.Encode([]byte(blockedDomains[i])))
+		//fmt.Println("Blocked domains[i]: ", blockedDomains[i])
+		//fmt.Println(e.Encode([]byte(blockedDomains[i])))
 		rappor = append(rappor, e.Encode([]byte(blockedDomains[i])))
 	}
-	err := json.NewEncoder(&buf).Encode(rappor)
+	svrMsg := SvrMsg{cohort, rappor}
+	err := json.NewEncoder(&buf).Encode(svrMsg)
 	if err != nil {
 		return err
 	}
@@ -524,6 +531,8 @@ func main() {
 	log.Println("Starting Metis proxy....")
 	//Set the HMAC secret for generating the PRRs in RAPPOR
 	generateSecret()
+	//TODO: Have the server tell the client what cohort it belongs to?
+	cohort=1
 	//Ask the master server for the blocked list
 	if getBlockedList() != nil {
 		log.Println("Error getting blocked list, starting with empty blocked list!")

server/rappor_analysis.py

@@ -35,6 +35,7 @@
 import sys
 import numpy as np
 import time
+import statsmodels.api as sm
 
 from random import SystemRandom
 from sklearn import linear_model
@@ -401,9 +402,10 @@ def getDomains(numDomains):
         domains.append(line.rstrip())
     return domains[0:numDomains]
 
-def getBloomFilters(domains, numCohorts, numHashes, numBloomBits):
+def getBloomFilters(domains, cohorts, numHashes, numBloomBits):
+    """cohorts is a list of the cohorts that were reported this time"""
     blooms = {}
-    for m in range(0, numCohorts):
+    for m in cohorts:
         bloomsForCohort = []
         for d in domains:
             bits = get_bloom_bits(d, m, numHashes, numBloomBits)
@@ -414,43 +416,51 @@ def getBloomFilters(domains, numCohorts, numHashes, numBloomBits):
     #print(blooms)
     return blooms
 
-def makeDesignMatrix(params, numDomains):
+def makeDesignMatrix(params, cohorts, numDomains):
+    """"cohorts is a list of the cohorts that were reported this time"""
     k = params.num_bloombits #number of bits in Bloom filter
-    m = params.num_cohorts
+    m = len(cohorts)
     #M is number of candidate strings
     #h is the number of hash functions per cohort
     domains = getDomains(numDomains)
-    blooms = getBloomFilters(domains, params.num_cohorts, params.num_hashes, params.num_bloombits)
+    blooms = getBloomFilters(domains, cohorts, params.num_hashes, params.num_bloombits)
 
     X = np.zeros([k*m, numDomains])
 
     for cohort in blooms:
         #print("Cohort: ", cohort)
         domainColumn = 0
+        rowChunk = 0
         for domain in blooms[cohort]:
             #domain is a list of the bits that need to be set
             for bitToSet in domain:
-                X[cohort*k+bitToSet, domainColumn] = 1
+                X[rowChunk*k+bitToSet, domainColumn] = 1
             domainColumn+=1
+        rowChunk += 1
     #print(X)
     return X, domains
 
 def doLassoRegression(params, numDomains, reports):
-    X, domains = makeDesignMatrix(params, numDomains)
+    #reports is supposed to be a dictionary whose keys are the cohort number and whose values are the list of reports from that cohort
+    #This needs to be rethought because every time we run this code, we're only using one cohort
+    #Need to redesign on client side to send cohort #
+    #So server needs to send cohort # when client first requests blocked list
+    X, domains = makeDesignMatrix(params, reports.keys(), numDomains)
     Y_list = []
-    for i in range(0, params.num_cohorts):
-        Y_j = estimateSetBits(reports[i], params)
+    for key in reports:
+        Y_j = estimateSetBits(reports[key], params)
         Y_list.append(Y_j)
     Y = np.array(Y_list)
     Y = Y.flatten()
 
+    print("********X_shape: ", X.shape, ", y-shape: ", Y.shape)
     linreg = linear_model.LassoCV(n_alphas=10, cv=10)
     linreg.fit(X,Y)
     print("Coefficients: ", linreg.coef_)
     print("Alpha: ", linreg.alpha_)
     return X, Y, linreg, domains
 
-def doLinearRegression(X, Y):
+def doLinearRegression(X, Y, M):
     X = sm.add_constant(X)
     ols = sm.OLS(Y,X)
     results = ols.fit()
@@ -482,19 +492,24 @@ def readRapporReports(filename):
     reportFile.close()
     return reps
 
-def analyzeReports(filename, params, numDomains):
-    reps = readRapporReports()
+def analyzeReports(reps, params, numDomains):
     lassX, lassY, lassoReg, domains = doLassoRegression(params, numDomains, reps)
     relevantDomains = np.where(lassoReg.coef_!=0)[0]
     linX = lassX[:,relevantDomains]
+    print("domains: ", domains, ", relevantDomains: ", relevantDomains)
     #Keep track of the domains the LASSO selected
-    linDoms = domains[relevantDomains]
-    #Confirm that the domains LASSO selected are relevant using linear regression
-    #This may prune out even more domains than LASSO did
-    finalRelevantIdxs = doLinearRegression(linX, lassY)
-    return linDoms[finalRelevantIdxs]
+    if len(relevantDomains) != 0:
+        doms = np.array(domains)
+        rels = np.array(relevantDomains)
+        linDoms = doms[rels]
+        #Confirm that the domains LASSO selected are relevant using linear regression
+        #This may prune out even more domains than LASSO did
+        finalRelevantIdxs = doLinearRegression(linX, lassY, numDomains)
+        return linDoms[finalRelevantIdxs]
+    else:
+        return []
 
-def main():
+"""def main():
     rapporRepsFile = "rappor_reports.txt"
     numDomains = 10
     params = Params(prob_f=0.2)
@@ -517,7 +532,7 @@ def main():
         mbl.write(d+"\r\n")
     
 
-
+"""
 
 
 

server/server.py

@@ -3,10 +3,12 @@
 
 from flask import Flask, request
 from flask_restful import Resource, Api, reqparse
-from json import dumps
 
+import json
 import sqlite3
 
+import rappor_analysis as rappor
+
 class Sql():
     def __init__(self, dbName):
         self.conn = sqlite3.connect(dbName)
@@ -18,33 +20,54 @@ def __init__(self, dbName):
         self.conn.commit()
 
     def populateDummyData(self):
-        # SQL command to insert the data in the table
         sql_command = """INSERT OR IGNORE INTO domains VALUES ("google.com");"""
         self.crsr.execute(sql_command)
+        sql_command = """INSERT OR IGNORE INTO domains VALUES ("facebook.com");"""
+        self.crsr.execute(sql_command)
+        self.conn.commit()
+        
+    def addToDB(self, domain):
+        cmd = "INSERT OR IGNORE INTO domains VALUES (\""+domain+"\");"
+        self.crsr.execute(cmd)
         self.conn.commit()
 
     def getBlockedList(self):
         cmd = "SELECT * FROM domains;"
         self.crsr.execute(cmd)
         return self.crsr.fetchall()
+    
+    def clearDB(self):
+        cmd = "DELETE FROM domains;"
+        self.crsr.execute(cmd)
+        self.conn.commit()
 
 
 class BlockedList(Resource):
     def __init__(self):
         self.sql = Sql("master_blocked_list.db")
+        #self.sql.clearDB()
         self.sql.populateDummyData()
 
     def get(self):
         print("Request for blocked list made by client: ")
-        blocked = self.sql.getBlockedList()
-        print(blocked)
-        #return [{'Domain':'Earth'}]
+        sqlBlocked = self.sql.getBlockedList()
+        blocked = []
+        for dom in sqlBlocked:
+            blocked.append({'Domain':dom[0]})
         return blocked
 
     def post(self):
         print("Client sent RAPPOR things to us: ")
         data = request.data
-        print "Data: ", data[0]
+        print(data)
+        cliMsg = json.loads(data)
+        print("Client message is ", cliMsg)
+        reps = {cliMsg["cohort"]:cliMsg["reports"]}
+        numDomains = 10
+        params = rappor.Params(prob_f=0.2)
+        doms = rappor.analyzeReports(reps, params, numDomains)
+        for d in doms:
+            self.sql.addToDB(d)
         return '', 200
 
 def main():