advanced Search, results obj

Favio Medrano · Favio Medrano · commit f193d2732131 · 2021-01-10T21:05:56.000-08:00
diff --git a/pyerddap/erddap_dataset.py b/pyerddap/erddap_dataset.py
@@ -2,9 +2,10 @@
 from pyerddap import url_operations
 from pyerddap.remote_requests import urlread
 from pyerddap.parse_utils import parseDictMetadata, parseConstraintValue
-from pyerddap.formatting import dataset_repr
+from pyerddap.formatting import dataset_repr, simple_dataset_repr
 import datetime as dt
 
+
 class ERDDAP_Dataset:
 
   DEFAULT_FILETYPE = 'csvp'
@@ -16,16 +17,20 @@ def __init__(self, erddapurl, datasetid, protocol='tabledap', auth=None, lazyloa
     self.protocol = protocol
     self.metadata = None
     self.erddapauth = auth
-    self.requestURL = None 
+
     self.resultVariables = []
     self.constraints = []
     self.serverSideFunctions = []
     self.variables = {} 
     if not lazyload:
       self.loadMetadata()
 
+
   def __repr__(self):
     return dataset_repr(self)
+
+  def __simple_repr__(self):
+    return simple_dataset_repr(self)
   
 
   def setResultVariables(self, variables):
@@ -38,14 +43,17 @@ def setResultVariables(self, variables):
     
     return self
 
+
   def addResultVariable(self, variable):
     self.resultVariables.append(variable)
     return self
 
+
   def setConstraints(self, constraintListOrDict):
     self.clearConstraints()
     self.addConstraints(constraintListOrDict)
 
+
   def addConstraints(self, constraintListOrDict):
     if isinstance(constraintListOrDict,dict):
       for k,v in constraintListOrDict.items():
@@ -57,6 +65,7 @@ def addConstraints(self, constraintListOrDict):
       raise Exception("Constraints argument must be either dictionary or list")
     return self
 
+
   def addConstraint(self, constraint):    
     if isinstance(constraint,dict):
       self._addConstraintDict(constraint)
@@ -65,10 +74,12 @@ def addConstraint(self, constraint):
     else:
       raise Exception("constraint argument must be either string or a dictionary")
     return self
-  
+
+
   def _addConstraintStr(self, constraint):
     self.constraints.append(constraint)
 
+
   def _addConstraintDict(self, constraintDict):
     constraintKey = next(iter(constraintDict))
     self._addConstraintStr(
@@ -126,12 +137,15 @@ def getMetadataURL(self, request_format='json'):
   def clearConstraints(self):
     self.constraints = []
   
+
   def clearServerSideFunctions(self):
     self.serverSideFunctions = []
   
+
   def clearResultVariables(self):
     self.resultVariables = []
 
+
   def clearQuery(self):
     self.clearConstraints()
     self.clearServerSideFunctions()
@@ -145,6 +159,7 @@ def getDataRequest(self, filetype=DEFAULT_FILETYPE, request_kwargs={}):
     else:
       return rawRequest.text
 
+
   # Server side functions wrappers
 
   def addVariablesWhere(self, attributeName, attributeValue):
diff --git a/pyerddap/erddap_server.py b/pyerddap/erddap_server.py
@@ -1,35 +1,107 @@
 import os
 from urllib.parse import quote_plus
 from pyerddap import url_operations
-from pyerddap.parse_utils import parseConstraintValue, parseConstraintDateTime
+from pyerddap.formatting import erddap_search_results_repr
+from pyerddap.parse_utils import parseConstraintValue, parseConstraintDateTime, parseSearchResults
 from pyerddap.remote_requests import urlread
 from pyerddap.erddap_dataset import ERDDAP_Dataset
+from pyerddap.erddap_tabledap import ERDDAP_Tabledap
+from pyerddap.erddap_griddap import ERDDAP_Griddap
+
+
+class ERDDAP_SearchResults(list):
+
+    def __repr__(self):
+        return erddap_search_results_repr(self)
+        
+    @property
+    def results(self):
+        return list(self)     
 
 class ERDDAP:
 
     ALLDATASETS_VARIABLES = ['datasetID','accessible','institution','dataStructure','cdm_data_type','class','title','minLongitude','maxLongitude','longitudeSpacing','minLatitude','maxLatitude','latitudeSpacing','minAltitude','maxAltitude','minTime','maxTime','timeSpacing','griddap','subset','tabledap','MakeAGraph','sos','wcs','wms','files','fgdc','iso19115','metadata','sourceUrl','infoUrl','rss','email','testOutOfDate','outOfDate','summary']
 
     def __init__(self, url, auth=None, lazyload=True):
         self.serverURL = url 
+        self.auth = auth
         self.tabledapAllDatasets = ERDDAP_Dataset(self.serverURL, 'allDatasets', auth=auth)
 
 
-    def getSearchURL(self, filetype='json', searchFor="", 
-                                            protocol="",
-                                            cdm_data_type="",
-                                            institution="",
-                                            ioos_category="",
-                                            keywords="",
-                                            long_name="",
-                                            standard_name="",
-                                            variableName="",
-                                            minLon="",
-                                            maxLon="",
-                                            minLat="",
-                                            maxLat=None,
-                                            minTime="",
-                                            maxTime="",
-                                            itemsPerPage=1000, page=1):
+    def advancedSearch(self, **filters):
+
+        searchURL = self.getSearchURL( **filters)
+        rawSearchResults = urlread(searchURL, self.auth)
+        dictSearchResult = rawSearchResults.json()
+        formatedResults = ERDDAP_SearchResults()
+
+        _griddap_dsets , _tabledap_dsets = parseSearchResults(dictSearchResult)
+        for dst in _tabledap_dsets:
+            formatedResults.append(ERDDAP_Tabledap(self.serverURL, dst, auth=self.auth))
+        for dst in _griddap_dsets:
+            formatedResults.append(ERDDAP_Griddap(self.serverURL, dst, auth=self.auth))
+
+        return formatedResults
+
+        
+
+
+    #def getSearchURL(self, filetype='json', searchFor="", 
+    #                                        protocol="",
+    #                                        cdm_data_type="",
+    #                                        institution="",
+    #                                        ioos_category="",
+    #                                        keywords="",
+    #                                        long_name="",
+    #                                        standard_name="",
+    #                                        variableName="",
+    #                                        minLon="",
+    #                                        maxLon="",
+    #                                        minLat="",
+    #                                        maxLat="",
+    #                                        minTime="",
+    #                                        maxTime="",
+    #                                        itemsPerPage=1000, page=1):
+    def getSearchURL(self, filetype='json', **searchFilters): 
+        """
+         Builds the url call for the advanced Search ERDDAP API Rest service.
+
+          Arguments
+           filetype   -  The result format (htmlTable, csv, json, tsv, etc)
+                         https://coastwatch.pfeg.noaa.gov/erddap/rest.html#responses 
+
+          Search filters:
+           searchFor     - This is a Google-like search of the datasets metadata, set the words you 
+                           want to search for  with spaces between the words.  ERDDAP will search 
+                           for the words separately, not as a phrase. 
+                           To search for a phrase, put double quotes around the phrase (for 
+                           example, "wind speed"). 
+                           To exclude datasets with a specific word, use -excludedWord. 
+                           To exclude datasets with a specific phrase, use -"excluded phrase" 
+                           To search for specific attribute values, use attName=attValue
+                           To find just grid or table datasets, include protocol=griddap 
+                           or protocol=tabledap
+
+           protocolol    - Set either: griddap, tabledap or wms                           
+           cdm_data_type -
+           institution   - 
+           ioos_category - 
+           keywords      - 
+           long_name     - 
+           standard_name - 
+           variableName  - 
+           minLon        -
+           maxLon        -
+           minLat        - 
+           maxLat        -
+           minTime       -
+           maxTime       -
+           itemsPerPage  -
+           page          -
+
+          Returns the string url for the search service.
+
+        """
 
         searchAPIEndpoint = "search/advanced.{}".format(filetype)
         searchAPIURL = os.path.join( self.serverURL, searchAPIEndpoint )
@@ -55,7 +127,8 @@ def getSearchURL(self, filetype='json', searchFor="",
 
         for queryElement, queryElementDefault in queryElementsDefaults.items():
                         
-            queryValue = eval(queryElement) if eval(queryElement) else queryElementDefault
+            #queryValue = eval(queryElement) if eval(queryElement) else queryElementDefault
+            queryValue = searchFilters.get(queryElement, queryElementDefault)
 
             if queryElement == 'searchFor':
                 if queryValue:
diff --git a/pyerddap/formatting.py b/pyerddap/formatting.py
@@ -1,5 +1,14 @@
 
 
+MAX_SUMMARY_LEN = 75
+
+def simple_dataset_repr(ds):
+    summary = ["<pyerddap.{}>".format(type(ds).__name__)]
+    dstTitle = ds.getAttribute('title')
+    truncatedTitle = (dstTitle[:MAX_SUMMARY_LEN] + '..') if len(dstTitle) > MAX_SUMMARY_LEN else dstTitle
+    summary.append("\"{}\"".format(truncatedTitle))
+    return ' '.join(summary)
+
 def dataset_repr(ds):
     summary = ["<pyerddap.{}>".format(type(ds).__name__)]
     summary.append("Title:       {}".format(ds.getAttribute('title')))
@@ -16,3 +25,12 @@ def dataset_repr(ds):
 
     return "\n".join(summary)
 
+
+def erddap_search_results_repr(srobj):
+    summary = ["<pyerddap.{}>".format(type(srobj).__name__)]
+    summary.append ("Results:  {}".format(len(list(srobj))))
+    summary.append('[')
+    for item in list(srobj):
+        summary.append( " " + item.__simple_repr__() )
+    summary.append(']')
+    return '\n'.join(summary)    
diff --git a/pyerddap/parse_utils.py b/pyerddap/parse_utils.py
@@ -1,7 +1,7 @@
 import re
 import datetime as dt
 
- 
+
 
 class ERDDAP_Metadata_Rows:
     ROW_TYPE       = 0
@@ -10,6 +10,26 @@ class ERDDAP_Metadata_Rows:
     DATA_TYPE      = 3
     VALUE          = 4
 
+class ERDDAP_Search_Results_Rows:
+    GRIDDAP     = 0
+    SUBSET      = 1
+    TABLEDAP    = 2
+    MAKEAGRAPH  = 3
+    WMS         = 4
+    FILES       = 5
+    ACCESIBLE   = 6
+    TITLE       = 7
+    SUMMARY     = 8
+    FGDC        = 9
+    ISO19115    = 10
+    INFO        = 11
+    BACKINFO    = 12
+    RSS         = 13
+    EMAIL       = 14
+    INSTITUTION = 15
+    DATASETID   = 16
+
+
 def parseDictMetadata(dmetadata):
     """
      This function parses the metadata json response from a erddap dataset
@@ -55,6 +75,19 @@ def parseMetadataAttribute(data_type, valuestr):
         return tuple(_castedvalue)
 
 
+def parseSearchResults(dresults):
+    _griddap_dsets = []
+    _tabledap_dsets = []
+    
+    for row in dresults['table']['rows']:
+        if row[ERDDAP_Search_Results_Rows.GRIDDAP]:
+            _griddap_dsets.append(row[ERDDAP_Search_Results_Rows.DATASETID])
+        elif row[ERDDAP_Search_Results_Rows.TABLEDAP]:
+            _tabledap_dsets.append(row[ERDDAP_Search_Results_Rows.DATASETID])
+
+    return _griddap_dsets, _tabledap_dsets
+
+
 def parseConstraintValue(value):
     """
      This functions detect the constraint value type and decide if is a 
diff --git a/pyerddap/url_operations.py b/pyerddap/url_operations.py
@@ -1,35 +1,7 @@
 import os 
 from urllib.parse import quote, quote_plus, urlparse, ParseResult
 
-
-#class URLBulder:
-#
-#    def __init__(self, base, query="", auth=(None,None)):
-#        self.baseurl = base
-#        self.query = query
-#        self.auth = auth
-#    
-#    def addQueryPair(self, query):
-#        if self._urlparseresult.query:
-#            self._urlparseresult.query = '&'.join([self._urlparseresult.query, query]) 
-#        else:
-#            self._urlparseresult.query = query
-#    
-#    def build(self, includeAuth=False):
-#        if includeAuth:
-#            return self._urlparseresult.geturl()
-#        else:
-#            return self._buildWithoutAuth()
-#            _authbk = (self._urlparseresult.username, self._urlparseresult.password)
-#            self._urlparseresult.username, self._urlparseresult.password = None, None
-#    
-#    def _buildWithoutAuth(self):
-#        _noauthparseresult = self._urlparseresult.copy()
-#        _noauthparseresult.username = None
-#        _noauthparseresult.password = None
-#        return _noauthparseresult.geturl()
         
-
 def parseQueryItems(items, useSafeURL=True, safe='', item_separator='&'):
     if useSafeURL:
         return quote(item_separator.join(items), safe=safe)
