url operations; parse utils ; erddap class

Author: Favio Medrano

pyerddap/erddap_dataset.py

@@ -1,8 +1,9 @@
 import os
-from urllib.parse import quote
+from pyerddap import url_operations
 from pyerddap.remote_requests import urlread
-from pyerddap.parse_utils import parseDictMetadata
+from pyerddap.parse_utils import parseDictMetadata, parseConstraintValue
 from pyerddap.formatting import dataset_repr
+import datetime as dt
 
 class ERDDAP_Dataset:
 
@@ -39,57 +40,71 @@ def setResultVariables(self, variables):
 
   def addResultVariable(self, variable):
     self.resultVariables.append(variable)
+    return self
 
-
-  def setConstraints(self, constraintList):
-    self.constraints = constraintList
-
-  def addConstraints(sef, constraintList):
-    for constraint in constraintList:
-      self.addConstraint(constraint)
+  def setConstraints(self, constraintListOrDict):
+    self.clearConstraints()
+    self.addConstraints(constraintListOrDict)
+
+  def addConstraints(self, constraintListOrDict):
+    if isinstance(constraintListOrDict,dict):
+      for k,v in constraintListOrDict.items():
+        self.addConstraint({k:v})
+    elif isinstance(constraintListOrDict,list):
+      for constraint in constraintListOrDict:
+        self.addConstraint(constraint)
+    else:
+      raise Exception("Constraints argument must be either dictionary or list")
     return self
 
-  def addConstraint(self, constraint):
-    self.constraints.append(constraint)
+  def addConstraint(self, constraint):    
+    if isinstance(constraint,dict):
+      self._addConstraintDict(constraint)
+    elif isinstance(constraint,str):
+      self._addConstraintStr(constraint)
+    else:
+      raise Exception("constraint argument must be either string or a dictionary")
     return self
+  
+  def _addConstraintStr(self, constraint):
+    self.constraints.append(constraint)
 
+  def _addConstraintDict(self, constraintDict):
+    constraintKey = next(iter(constraintDict))
+    self._addConstraintStr(
+      "{key_plus_conditional}{value}".format(
+            key_plus_conditional=constraintKey, 
+            value=parseConstraintValue(constraintDict[constraintKey])
+          )
+    )
 
-  def getDataRequestURL(self, filetype=DEFAULT_FILETYPE, isQuoted=True):
-    
-    requestURL = self.getDownloadURL(filetype)
+
+  def getDataRequestURL(self, filetype=DEFAULT_FILETYPE, useSafeURL=True):
+    requestURL = self.getBaseURL(filetype)
     query = ""
 
     if len(self.resultVariables) > 0:
-      query += self.parseQueryItems(self.resultVariables, isQuoted, safe='', argument_separator=',')
+      query += url_operations.parseQueryItems(self.resultVariables, useSafeURL, safe='', item_separator=',')
 
     if len(self.constraints) > 0:
-      query += '&' + self.parseQueryItems(self.constraints, isQuoted, safe='=!()&')
+      query += '&' + url_operations.parseQueryItems(self.constraints, useSafeURL, safe='=!()&')
 
     if len(self.serverSideFunctions) > 0:
-      query += '&' + self.parseQueryItems(self.serverSideFunctions, isQuoted, safe='=!()&/')
+      query += '&' + url_operations.parseQueryItems(self.serverSideFunctions, useSafeURL, safe='=!()&/')
 
-    if len(query)>0:
-      requestURL += '?' + query
+    requestURL = url_operations.joinURLElements(requestURL, query)
 
     self.lastRequestURL = requestURL
     return self.lastRequestURL
 
 
-  def parseQueryItems(self, items, isQuoted=True, safe='', argument_separator='&'):
-    if isQuoted:
-      return quote(argument_separator.join(items), safe=safe)
-    else:
-      return argument_separator.join(items)
-
-
-  def getDownloadURL(self, filetype=DEFAULT_FILETYPE):
+  def getBaseURL(self, filetype=DEFAULT_FILETYPE):
     if filetype.lower() == 'opendap':
       return os.path.join(self.erddapurl, self.protocol, self.datasetid )
     else:
       return os.path.join(self.erddapurl, self.protocol, self.datasetid + "." + filetype )
 
 
-
   def getAttribute(self, attribute, variableName='NC_GLOBAL'):
     self.loadMetadata()
     for rowAttribute in self.metadata:
@@ -136,7 +151,9 @@ def addVariablesWhere(self, attributeName, attributeValue):
     '''
      https://coastwatch.pfeg.noaa.gov/erddap/tabledap/documentation.html#addVariablesWhere
     '''
-    self.serverSideFunctions.append( 'addVariablesWhere("%s","%s")' % (attributeName, attributeValue) )
+    self.serverSideFunctions.append( 
+      'addVariablesWhere("{}","{}")'.format(attributeName, attributeValue) 
+    )
     return self
 
   def distinct(self):
@@ -150,7 +167,7 @@ def units(self, value):
     '''
      https://coastwatch.pfeg.noaa.gov/erddap/tabledap/documentation.html#units
     '''
-    self.serverSideFunctions.append( 'units(%s)' % value )
+    self.serverSideFunctions.append( 'units({})'.format(value) )
 
   def orderBy(self, variables):
     '''
@@ -209,7 +226,11 @@ def orderByMean(self, variables):
     return self     
 
   def addServerSideFunction(self, functionName, arguments):
-    self.serverSideFunctions.append( '%s("%s")' % ( functionName, self.parseListOrStrToCommaSeparatedString(arguments) ) )
+    self.serverSideFunctions.append(
+      "{}(\"{}\")".format(
+        functionName, self.parseListOrStrToCommaSeparatedString(arguments)
+      )
+    )
 
 
   def parseListOrStrToCommaSeparatedString(self, listorstring):

pyerddap/erddap_server.py

@@ -0,0 +1,88 @@
+import os
+from urllib.parse import quote_plus
+from pyerddap import url_operations
+from pyerddap.parse_utils import parseConstraintValue, parseConstraintDateTime
+from pyerddap.remote_requests import urlread
+from pyerddap.erddap_dataset import ERDDAP_Dataset
+
+class ERDDAP:
+
+    ALLDATASETS_VARIABLES = ['datasetID','accessible','institution','dataStructure','cdm_data_type','class','title','minLongitude','maxLongitude','longitudeSpacing','minLatitude','maxLatitude','latitudeSpacing','minAltitude','maxAltitude','minTime','maxTime','timeSpacing','griddap','subset','tabledap','MakeAGraph','sos','wcs','wms','files','fgdc','iso19115','metadata','sourceUrl','infoUrl','rss','email','testOutOfDate','outOfDate','summary']
+
+    def __init__(self, url, auth=None, lazyload=True):
+        self.serverURL = url 
+        self.tabledapAllDatasets = ERDDAP_Dataset(self.serverURL, 'allDatasets', auth=auth)
+
+
+    def getSearchURL(self, filetype='json', searchFor="", 
+                                            protocol="",
+                                            cdm_data_type="",
+                                            institution="",
+                                            ioos_category="",
+                                            keywords="",
+                                            long_name="",
+                                            standard_name="",
+                                            variableName="",
+                                            minLon="",
+                                            maxLon="",
+                                            minLat="",
+                                            maxLat=None,
+                                            minTime="",
+                                            maxTime="",
+                                            itemsPerPage=1000, page=1):
+
+        searchAPIEndpoint = "search/advanced.{}".format(filetype)
+        searchAPIURL = os.path.join( self.serverURL, searchAPIEndpoint )
+
+        queryElementsDefaults = { 'page'          : 1 ,
+                                  'itemsPerPage'  : 1000,
+                                  'searchFor'     : None,
+                                  'protocol'      : "(ANY)",
+                                  'cdm_data_type' : "(ANY)",
+                                  'institution'   : "(ANY)",
+                                  'ioos_category' : "(ANY)",
+                                  'keywords'      : "(ANY)",
+                                  'long_name'     : "(ANY)",
+                                  'standard_name' : "(ANY)",
+                                  'variableName'  : "(ANY)",
+                                  'maxLat'        : None,
+                                  'minLon'        : None,
+                                  'maxLon'        : None,
+                                  'minLat'        : None,
+                                  'minTime'       : None,
+                                  'maxTime'       : None}
+        queryURL=[]
+
+        for queryElement, queryElementDefault in queryElementsDefaults.items():
+                        
+            queryValue = eval(queryElement) if eval(queryElement) else queryElementDefault
+
+            if queryElement == 'searchFor':
+                if queryValue:
+                    queryValue = quote_plus(queryValue)
+                queryURL.append( queryElement + "=" + ("" if queryValue is None else queryValue) )
+                continue
+ 
+            if queryValue is None:
+                queryURL.append( queryElement + "=" )      
+            elif queryElement in ['minTime', 'maxTime']:
+                queryURL.append( queryElement + "=" + parseConstraintDateTime(queryValue) )
+            else: 
+                queryURL.append( queryElement + "=" + str(queryValue) )
+        
+        return url_operations.joinURLElements(searchAPIURL, url_operations.parseQueryItems(queryURL, safe='=+-&'))
+        #return searchAPIURL + "?" + "&".join(queryURL)
+
+
+    def getQueryAllDatasetsURL(self, filetype='json', constraints=[]):
+
+        resultVariables = self.ALLDATASETS_VARIABLES
+        response = (
+            self.tabledapAllDatasets.setResultVariables(resultVariables)
+                                    .setConstraints(constraints)
+                                    .getDataRequestURL(filetype=filetype)
+        )
+        return response
+
+
+        

pyerddap/formatting.py

@@ -8,6 +8,11 @@ def dataset_repr(ds):
     summary.append("Variables: ")
     for variableName, variableAttributes in ds.variables.items():
         summary.append("  {} ({}) ".format(variableName, variableAttributes['data_type']) )
+        if 'standard_name' in variableAttributes:
+            summary.append("    Standard name: {} ".format(variableAttributes['standard_name']) )
+        if 'units' in variableAttributes:
+            summary.append("    Units:         {} ".format(variableAttributes['units']) )
+
 
     return "\n".join(summary)
 

pyerddap/parse_utils.py

@@ -1,4 +1,7 @@
+import re
+import datetime as dt
 
+ 
 
 class ERDDAP_Metadata_Rows:
     ROW_TYPE       = 0
@@ -7,7 +10,6 @@ class ERDDAP_Metadata_Rows:
     DATA_TYPE      = 3
     VALUE          = 4
 
-
 def parseDictMetadata(dmetadata):
     """
      This function parses the metadata json response from a erddap dataset
@@ -41,11 +43,80 @@ def parseMetadataAttribute(data_type, valuestr):
         _lvaluestr = [valuestr]
 
     if data_type in ['float', 'double']:
-        _castedvalue =  [float(v) for v in _lvaluestr] 
+        _castedvalue =  [ float(v) for v in _lvaluestr ]
     elif data_type in ['short', 'int', 'byte', 'char', 'short', 'long']:
-        _castedvalue =  [int(v) for v in _lvaluestr] 
+        _castedvalue =  [ int(v) for v in _lvaluestr ]
+    else:
+        _castedvalue =  [ v for v in _lvaluestr ]
 
     if len(_castedvalue) == 1:
         return _castedvalue[0]
     else:
-        return _castedvalue
+        return tuple(_castedvalue)
+
+
+def parseConstraintValue(value):
+    """
+     This functions detect the constraint value type and decide if is a 
+     regular string and if so, put quotes "" around it. 
+     Detect if the constraint value is either: 
+      String value         : Return the string inside quotes "<value>"
+      <python.datetime>    : Convert to string date with format ISO 8601
+      Valid ISO 8601 Date  : Return the string date
+      Valid time operation : Return the string operation
+      Valid variable 
+      operation            : Return the string operation
+    """
+    if isinstance(value, str):    
+        if validate_iso8601(value):
+            return value 
+        if validate_constraint_time_operations(value):
+            return value
+        if validate_constraint_var_operations(value):
+            return value
+        else:
+            return '"{}"'.format(value)
+    elif isinstance(value,dt.datetime):
+        return parseConstraintPyDatetime(value)
+    else:
+        return str(value)
+
+def parseConstraintDateTime(dtvalue):
+    if isinstance(dtvalue,dt.datetime):
+        return parseConstraintPyDatetime(dtvalue)
+    elif isinstance(dtvalue, str):
+        return dtvalue
+
+def parseConstraintPyDatetime(dtvalue):
+    return dtvalue.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+#DATE_ISO8601_REGEX = r'^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?(Z|[+-](?:2[0-3]|[01][0-9]):[0-5][0-9])?$'
+
+# Regular expression validators
+DATE_ISO8601_REGEX = r'^\d{4}(-\d\d(-\d\d(T\d\d(:\d\d)?(:\d\d)?(\.\d+)?(([+-]\d\d:\d\d)|Z)?)?)?)?$'
+CONSTRAINT_TIME_OPERATIONS_REGEX = r'^(max|min)\(\w(\w|\d)*\)((-|\+)\d+(millis|seconds|minutes|hours|days|months|years))?$'
+CONSTRAINT_VAR_OPERATIONS_REGEX = r'^(max|min)\(\w(\w|\d)*\)((-|\+)\d+(.\d+)?)?$' 
+
+match_timeoper = re.compile(CONSTRAINT_TIME_OPERATIONS_REGEX).match   
+match_iso8601 = re.compile(DATE_ISO8601_REGEX).match
+match_varoper = re.compile(CONSTRAINT_VAR_OPERATIONS_REGEX).match   
+
+# https://stackoverflow.com/questions/41129921/validate-an-iso-8601-datetime-string-in-python
+# https://stackoverflow.com/questions/12756159/regex-and-iso8601-formatted-datetime
+def validate_iso8601(str_val):    
+    return validateRegex(str_val, match_iso8601)
+
+def validate_constraint_time_operations(str_val):
+    return validateRegex(str_val, match_timeoper)
+
+def validate_constraint_var_operations(str_val):
+    return validateRegex(str_val, match_varoper)
+
+def validateRegex(str_val, rematch):
+    try:
+        if rematch(str_val) is not None:
+            return True
+    except:
+        pass
+    return False    

pyerddap/url_operations.py

@@ -0,0 +1,47 @@
+import os 
+from urllib.parse import quote, quote_plus, urlparse, ParseResult
+
+
+#class URLBulder:
+#
+#    def __init__(self, base, query="", auth=(None,None)):
+#        self.baseurl = base
+#        self.query = query
+#        self.auth = auth
+#    
+#    def addQueryPair(self, query):
+#        if self._urlparseresult.query:
+#            self._urlparseresult.query = '&'.join([self._urlparseresult.query, query]) 
+#        else:
+#            self._urlparseresult.query = query
+#    
+#    def build(self, includeAuth=False):
+#        if includeAuth:
+#            return self._urlparseresult.geturl()
+#        else:
+#            return self._buildWithoutAuth()
+#            _authbk = (self._urlparseresult.username, self._urlparseresult.password)
+#            self._urlparseresult.username, self._urlparseresult.password = None, None
+#    
+#    def _buildWithoutAuth(self):
+#        _noauthparseresult = self._urlparseresult.copy()
+#        _noauthparseresult.username = None
+#        _noauthparseresult.password = None
+#        return _noauthparseresult.geturl()
+        
+
+def parseQueryItems(items, useSafeURL=True, safe='', item_separator='&'):
+    if useSafeURL:
+        return quote(item_separator.join(items), safe=safe)
+    else:
+        return item_separator.join(items)
+
+def joinURLElements(base, query):
+    return base + '?' + query
+
+def joinURLElementsWithAuth(base, query, auth):
+    abase = base.replace("https://", "https://{}:{}@".format(auth[0],auth[1]))
+    abase = base.replace("http://", "http://{}:{}@".format(auth[0],auth[1]))
+    return joinURLElements(abase, query)
+
+