Merge branch 'v1': Finally moving to Version 1

README.md

@@ -2,15 +2,18 @@ Searsia Server
 ==============
 http://searsia.org
 
-Usage: 
+Usage:
 + Build with: `mvn package`
-+ Run with: `java -jar target/searsiaserver.jar`
++ Run with: `java -jar target/searsiaserver.jar -m <url>`
 + Done.
 
-Connect to the server with the [Federated Web Search Client][1].
-More information can be found in the [Searsia Documentation][2], 
-or you may ask a question under [Searsia Server Issues][3].
+The option `-m` is required: It connects your server to an
+existing Searsia server, see [Searsia server options][1].
+Connect to your server with the [Federated Web Search Client][2].
+More information can be found in the [Searsia Documentation][3],
+or you may ask a question under [Searsia Server Issues][4].
 
-[1]: http://github.com/searsia/searsiaclient "Searsia Client"
-[2]: http://searsia.org "Searsia Documentation"
-[3]: http://github.com/searsia/searsiaserver/issues "Issues"
+[1]: http://searsia.org/start.html#server
+[2]: http://github.com/searsia/searsiaclient "Searsia Client"
+[3]: http://searsia.org "Searsia Documentation"
+[4]: http://github.com/searsia/searsiaserver/issues "Issues"

pom.xml

@@ -3,9 +3,9 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.searsia</groupId>
   <artifactId>searsiaserver</artifactId>
-  <version>0.4.1</version>
+  <version>1.0.2</version>
   <prerequisites>
-    <maven>3</maven>
+    <maven>3.0</maven>
   </prerequisites>
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -19,7 +19,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-assembly-plugin</artifactId>
-        <version>2.6</version>
+        <version>3.1.0</version>
         <configuration>
           <descriptorRefs>
             <descriptorRef>jar-with-dependencies</descriptorRef>

src/main/java/org/searsia/Hit.java

@@ -23,6 +23,13 @@
 
 import org.json.JSONObject;
 
+/**
+ * A single search hit. A hit can have any field. 
+ * Standard fields are "title", "description", "url, "favicon", "image".
+ * 
+ * @author Djoerd Hiemstra and Dolf Trieschnigg
+ */
+
 public class Hit implements Comparable<Hit> {
 
 	private Map<String,Object> map;
@@ -44,7 +51,12 @@ public Hit(JSONObject json) {
 		Iterator<?> keys = json.keys();
 		while (keys.hasNext()) {
 			String key = (String) keys.next();
-			map.put(key, json.get(key));
+			Object value = json.get(key);
+			if (value instanceof String) {
+    			map.put(key, noHTML((String) value));
+			} else if (value instanceof Number || value instanceof Boolean) {
+				map.put(key, value);
+			}
 		}
 	}
 
@@ -71,10 +83,10 @@ public void setScore(float score) {
 		map.put("score", score);
 	}
 
-	public void setRank(int rank) {
-		map.put("rank", rank);
-	}
-	
+    public void setResourceScore(float score) {
+        map.put("rscore", score);
+    }
+    
 	public void setTitle(String title) {
 		map.put("title", title);
 	}
@@ -87,27 +99,52 @@ public void setUrl(String url) {
 		map.put("url", url);
 	}
 
+	/**
+	 * This id of will be used the Lucene index.
+	 * One url may be indexed multiple times, 
+	 * once for each resource id (rid).
+	 * @return unique identifier
+	 */
 	public String getId() {
 		String result = (String) map.get("url");
+		String rid = "";
 		if (result == null) {
 			result = (String) map.get("title");
-		}
-		return result;
-	}
-
-	public float getScore() {
-		Float score = (Float) map.get("score");
-		if (score == null) {
-			return 0.0f;
 		} else {
-			return score;
+            rid = (String) map.get("rid");
+            if (rid == null) {
+                rid = "";
+            }
 		}
+		return rid + "@" + result;
 	}
-
-	public Integer getRank() {
-		return (Integer) map.get("rank");
+
+	private float getFloatValue(String field) {
+        Float score = 0.0f;
+        Object scoreObject = map.get(field);
+        if (scoreObject instanceof Float) {
+            score = (float) scoreObject;
+        } else if (scoreObject instanceof Double) {
+            score = new Float((double) scoreObject); 
+        } else if (scoreObject instanceof Integer) {
+            score = new Float((int) scoreObject); 
+        } else if (scoreObject instanceof String) {
+            try {
+                score = Float.parseFloat((String) scoreObject);
+            } catch (NumberFormatException e) { }
+        } 
+        return score;
+	}
+
+
+	public float getScore() {
+	    return getFloatValue("score");
 	}
 
+    public float getResourceScore() {
+        return getFloatValue("rscore");
+    }
+
 	public Object get(String field) {
 		return map.get(field);
 	}
@@ -124,29 +161,50 @@ public String getTitle() {
 		return (String) map.get("title");
 	}
 
+    public String getRid() {
+        return (String) map.get("rid");
+    }
+
 	@Override
 	public String toString() {
 		return map.entrySet().toString();
 	}
 
-	private String noHTML(String value) {
-		value = value.replaceAll("<[^>]+>", ""); // no HTML
+	private String noHTML(String value) {  // TODO: also in TextExtractor??
+		value = value.replaceAll("(?i)</?span[^>]*>|</?b>|</?i>|</?em>|</?strong>", "");  // No HTML, please: spans removed 
+		value = value.replaceAll("<[^>]+>|&#?[0-9a-zA-Z]{1,9};", ""); // no HTML
 		return value.replaceAll("[<>]", "");
 	}
 
-	public JSONObject toJson() {
+    public JSONObject toJson() {
+        JSONObject json = new JSONObject();
+        for (Entry<String,Object> e: map.entrySet()) {
+            Object value = e.getValue();
+            if (value instanceof String) {
+                value = noHTML((String) value);
+            }
+            json.put(e.getKey(), value);
+        }
+        return json;
+    }
+
+
+	public JSONObject toJsonNoQueryResourceId() {
 		JSONObject json = new JSONObject();
 		for (Entry<String,Object> e: map.entrySet()) {
 			Object value = e.getValue();
 			if (value instanceof String) {
 				value = noHTML((String) value);
 			}
-			json.put(e.getKey(), value);
+			String key = e.getKey();
+			if (!key.equals("query") && !key.equals("rid")) {
+                json.put(key, value);
+			}
 		}
 		return json;
 	}
 
-    public String toIndexVersion() { // TODO: special treatment for urls, etc.
+    public String toIndexVersion() { // TODO: special treatment for urls, etc. and StringBuilder
     	String result = "";
     	for (Object s : map.values()) {
     		if (s instanceof String) {
@@ -156,20 +214,36 @@ public String toIndexVersion() { // TODO: special treatment for urls, etc.
     	return result.trim();
     }
 
+    public String toTitleDescriptionIndexVersion() {
+        String result = (String) this.get("title");
+        String desc   = (String) this.get("description");
+        if (result == null) { result = ""; }
+        if (desc != null) {
+            result += " " + desc;
+        }
+        return result.trim();
+    }
+
     @Override
     public int compareTo(Hit hit2) {
-    	Float score1 = getScore();
-    	Float score2 = hit2.getScore();
-    	if (score1.compareTo(score2) == 0) {
-    		Integer rank1 = getRank();
-    		Integer rank2 = hit2.getRank();
-    		if (rank1 != null && rank2 != null) {
-        		return rank2.compareTo(rank1); // yes reversed!
-    		} else {
-    			return 0;
-    		}
+    	Float score1 = getResourceScore();  // order by best resources
+    	Float score2 = hit2.getResourceScore();
+    	int compare = score1.compareTo(score2);
+    	if (compare != 0) {
+            return compare;
     	} else {
-    		return score1.compareTo(score2);
+    	    String rid1 = getRid(); // if two resources the same score
+    	    String rid2 = hit2.getRid();
+    	    if (rid1 != null && rid2 != null) {
+    	        compare = rid1.compareTo(rid2);
+    	    }
+    	    if (compare != 0) {
+    	        return compare;
+    	    } else {
+        		score1 = getScore(); // cannot be null
+        		score2 = hit2.getScore();
+      		    return score1.compareTo(score2);
+    		}
     	}
     }