matchingSet built using OpenSearch

src/main/java/org/ecocean/Encounter.java

@@ -4363,20 +4363,35 @@ public void run() {
                 bgShepherd.beginDBTransaction();
                 try {
                     Encounter enc = bgShepherd.getEncounter(encId);
-                    if ((enc == null) || !enc.hasMarkedIndividual()) {
+                    if ((enc == null) || (!enc.hasMarkedIndividual() && !enc.hasAnnotations())) {
                         // bgShepherd.rollbackAndClose();
                         executor.shutdown();
                         return;
                     }
                     MarkedIndividual indiv = enc.getIndividual();
-                    System.out.println("opensearchIndexDeep() background indexing indiv " +
-                        indiv.getId() + " via enc " + encId);
-                    try {
-                        indiv.opensearchIndex();
-                    } catch (Exception ex) {
-                        System.out.println("opensearchIndexDeep() background indexing " +
-                            indiv.getId() + " FAILED: " + ex.toString());
-                        ex.printStackTrace();
+                    if (indiv != null) {
+                        System.out.println("opensearchIndexDeep() background indexing indiv " +
+                            indiv.getId() + " via enc " + encId);
+                        try {
+                            indiv.opensearchIndex();
+                        } catch (Exception ex) {
+                            System.out.println("opensearchIndexDeep() background indexing " +
+                                indiv.getId() + " FAILED: " + ex.toString());
+                            ex.printStackTrace();
+                        }
+                    }
+                    if (enc.hasAnnotations()) {
+                        for (Annotation ann : enc.getAnnotations()) {
+                            System.out.println("opensearchIndexDeep() background indexing annot " +
+                                ann.getId() + " via enc " + encId);
+                            try {
+                                ann.opensearchIndex();
+                            } catch (Exception ex) {
+                                System.out.println("opensearchIndexDeep() background indexing " +
+                                    ann.getId() + " FAILED: " + ex.toString());
+                                ex.printStackTrace();
+                            }
+                        }
                     }
                 } catch (Exception e) {
                     System.out.println("opensearchIndexDeep() backgrounding Encounter " + encId +
@@ -4390,7 +4405,7 @@ public void run() {
                 executor.shutdown();
             }
         };
-        System.out.println("opensearchIndexDeep() begin backgrounding indiv for " + this);
+        System.out.println("opensearchIndexDeep() begin backgrounding for " + this);
         executor.execute(rn);
         System.out.println("opensearchIndexDeep() [foreground] finished for Encounter " + encId);
     }

src/main/java/org/ecocean/OpenSearch.java

@@ -397,6 +397,25 @@ public JSONObject queryPit(String indexName, final JSONObject query, int numFrom
         return new JSONObject(rtn);
     }
 
+    // just return the actual hit results
+    // note: each object in the array has _id but actual doc is in _source!!
+    public static JSONArray getHits(JSONObject queryResults) {
+        JSONArray failed = new JSONArray();
+
+        if (queryResults == null) return failed;
+        JSONObject outerHits = queryResults.optJSONObject("hits");
+        if (outerHits == null) {
+            System.out.println("could not find (outer) hits");
+            return failed;
+        }
+        JSONArray hits = outerHits.optJSONArray("hits");
+        if (hits == null) {
+            System.out.println("could not find hits");
+            return failed;
+        }
+        return hits;
+    }
+
     // https://opensearch.org/docs/2.3/opensearch/search/paginate/
     public JSONObject queryRawScroll(String indexName, final JSONObject query, int pageSize)
     throws IOException {

src/main/java/org/ecocean/Shepherd.java

@@ -2063,6 +2063,26 @@ public Iterator getAllAnnotationsNoQuery() {
         }
     }
 
+    // note: where clause can also contain " ORDER BY xxx"
+    public Iterator getAnnotationsFilter(String jdoWhereClause) {
+        Query query = null;
+
+        try {
+            query = pm.newQuery("SELECT FROM org.ecocean.Annotation WHERE " + jdoWhereClause);
+            Collection c = (Collection)(query.execute());
+            List list = new ArrayList(c);
+            Iterator it = list.iterator();
+            query.closeAll();
+            return it;
+        } catch (Exception npe) {
+            System.out.println(
+                "Error encountered when trying to execute getAllAnnotationsFilter. Returning a null iterator.");
+            npe.printStackTrace();
+            if (query != null) query.closeAll();
+            return null;
+        }
+    }
+
     public Iterator<Encounter> getAllAnnotations(String order) {
         Extent extClass = pm.getExtent(Annotation.class, true);
         Query q = pm.newQuery(extClass);

src/main/java/org/ecocean/ia/plugin/WildbookIAM.java

@@ -11,10 +11,10 @@
 import javax.servlet.ServletContextEvent;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.ecocean.acm.AcmUtil;
+import org.ecocean.Annotation;
 import org.ecocean.cache.CachedQuery;
 import org.ecocean.cache.QueryCache;
 import org.ecocean.cache.QueryCacheFactory;
-import org.ecocean.Annotation;
 import org.ecocean.ia.IA;
 import org.ecocean.ia.Task;
 import org.ecocean.media.*;
@@ -82,48 +82,6 @@ public void prime() {
         IBEISIA.setIAPrimed(true);
     }
 
-    public void _OLD_prime() {
-        IA.log("INFO: WildbookIAM.prime(" + this.context + ") called");
-        IBEISIA.setIAPrimed(false);
-        if (!isEnabled()) return;
-        final List<String> iaAnnotIds = iaAnnotationIds();
-        final List<String> iaImageIds = iaImageIds();
-        Runnable r = new Runnable() {
-            public void run() {
-                Shepherd myShepherd = new Shepherd(context);
-                myShepherd.setAction("WildbookIAM.prime");
-                myShepherd.beginDBTransaction();
-                ArrayList<Annotation> matchingSet = Annotation.getAllMatchAgainstTrue(myShepherd);
-                ArrayList<Annotation> sendAnns = new ArrayList<Annotation>();
-                ArrayList<MediaAsset> mas = new ArrayList<MediaAsset>();
-                for (Annotation ann : matchingSet) {
-                    if (iaAnnotIds.contains(ann.getAcmId())) continue; // no need to send
-                    sendAnns.add(ann);
-                    MediaAsset ma = ann.getDerivedMediaAsset();
-                    if (ma == null) ma = ann.getMediaAsset();
-                    if (ma == null) continue;
-                    if (iaImageIds.contains(ma.getAcmId())) continue;
-                    mas.add(ma);
-                }
-                IA.log("INFO: WildbookIAM.prime(" + context + ") sending " + sendAnns.size() +
-                    " annots (of " + matchingSet.size() + ") and " + mas.size() + " images");
-                try {
-                    // think we can checkFirst on both of these -- no need to re-send anything during priming
-                    sendMediaAssets(mas, true);
-                    sendAnnotations(sendAnns, true, myShepherd);
-                } catch (Exception ex) {
-                    IA.log("ERROR: WildbookIAM.prime() failed due to " + ex.toString());
-                    ex.printStackTrace();
-                }
-                myShepherd.commitDBTransaction(); // MAs and annots may have had acmIds changed
-                myShepherd.closeDBTransaction();
-                IBEISIA.setIAPrimed(true);
-                IA.log("INFO: WildbookIAM.prime(" + context + ") complete");
-            }
-        };
-        new Thread(r).start();
-    }
-
 /*
     note: sendMediaAssets() and sendAnnotations() need to be *batched* now in small chunks, particularly sendMediaAssets().
     this is because we **must** get the return value from the POST, in order that we can map the corresponding (returned) acmId values.  if we
@@ -317,33 +275,28 @@ public List<String> iaAnnotationIds() {
     public static List<String> iaAnnotationIds(String context) {
         List<String> ids = new ArrayList<String>();
         JSONArray jids = null;
-        String cacheName="iaAnnotationIds";
+        String cacheName = "iaAnnotationIds";
+
         try {
-
-
-        	QueryCache qc = QueryCacheFactory.getQueryCache(context);
+            QueryCache qc = QueryCacheFactory.getQueryCache(context);
             if (qc.getQueryByName(cacheName) != null &&
                 System.currentTimeMillis() <
                 qc.getQueryByName(cacheName).getNextExpirationTimeout()) {
-
-            	     org.datanucleus.api.rest.orgjson.JSONObject jobj = Util.toggleJSONObject(qc.getQueryByName(cacheName).getJSONSerializedQueryResult());
-            	     jids=Util.toggleJSONArray(jobj.getJSONArray("iaAnnotationIds"));
-            }
-            else {
-            	jids = apiGetJSONArray("/api/annot/json/", context);
-            	if(jids!=null) {
-	            	org.datanucleus.api.rest.orgjson.JSONObject jobj =new org.datanucleus.api.rest.orgjson.JSONObject();
-	            	jobj.put("iaAnnotationIds",Util.toggleJSONArray(jids));
-	            	CachedQuery cq = new CachedQuery(cacheName, Util.toggleJSONObject(jobj));
-	                cq.nextExpirationTimeout = System.currentTimeMillis() + (15*60*1000);
-	                qc.addCachedQuery(cq);
-            	}
-
+                org.datanucleus.api.rest.orgjson.JSONObject jobj = Util.toggleJSONObject(
+                    qc.getQueryByName(cacheName).getJSONSerializedQueryResult());
+                jids = Util.toggleJSONArray(jobj.getJSONArray("iaAnnotationIds"));
+            } else {
+                jids = apiGetJSONArray("/api/annot/json/", context);
+                if (jids != null) {
+                    org.datanucleus.api.rest.orgjson.JSONObject jobj =
+                        new org.datanucleus.api.rest.orgjson.JSONObject();
+                    jobj.put("iaAnnotationIds", Util.toggleJSONArray(jids));
+                    CachedQuery cq = new CachedQuery(cacheName, Util.toggleJSONObject(jobj));
+                    cq.nextExpirationTimeout = System.currentTimeMillis() + (15 * 60 * 1000);
+                    qc.addCachedQuery(cq);
+                }
             }
-
-
-        } 
-        catch (Exception ex) {
+        } catch (Exception ex) {
             ex.printStackTrace();
             IA.log("ERROR: WildbookIAM.iaAnnotationIds() returning empty; failed due to " +
                 ex.toString());

src/main/java/org/ecocean/identity/IAQueryCache.java

@@ -25,7 +25,7 @@ public static int buildTargetAnnotationsCache(String context, ArrayList<Annotati
         // qanns can "in theory" contain more than one annot, but we arent ready for that universe yet anyway...
         if ((qanns == null) || (qanns.size() < 1)) return -1; // :(
         // we want *non* excluding version of this:
-        ArrayList<Annotation> anns = qanns.get(0).getMatchingSetForTaxonomy(myShepherd, null);
+        ArrayList<Annotation> anns = qanns.get(0).getMatchingSet(myShepherd);
         if (anns == null) return -2;
         JSONObject jdata = new JSONObject();
         JSONArray idArr = new JSONArray();
@@ -149,7 +149,6 @@ public static JSONObject addTargetAnnotation(String context, Annotation ann,
             log("WARNING: addTargetAnnotation() found empty current arrays!  weird.");
             return null;
         }
-
         String indivId = ann.findIndividualId(myShepherd);
         // cheap fix to handle name-conflict potential: we dont add an annot which is already on list in cache!
         if (indivId == null) {

src/main/webapp/appadmin/opensearchSync.jsp

@@ -88,7 +88,7 @@ if (endNum > 0) {
     if (indexName.equals("encounter")) {
         itr = myShepherd.getAllEncounters("catalogNumber");
     } else if (indexName.equals("annotation")) {
-        itr = myShepherd.getAllAnnotations("id");
+        itr = myShepherd.getAnnotationsFilter("matchAgainst == true && acmId != null ORDER BY id");
     } else if (indexName.equals("individual")) {
         itr = myShepherd.getAllMarkedIndividuals();
     }