Merge pull request #167 from coecms/docsaug23

Docsaug23

Author: paolap

Makefile

@@ -1,4 +1,4 @@
-ENV=module load conda;
+ENV=module load conda/analysis3-23.04;
 SHELL=/bin/bash
 
 .PHONY: check test package

clef/cli.py

@@ -371,7 +371,7 @@ def common_esgf_cli(ctx, project, query, latest, replica, distrib,
                 print(f"{row.Index[0]} / {row.Index[1]} versions: {', '.join([v[0] for v in row.version])}")
             ids = get_ids(results) 
         else:
-            q = find_checksum_id(' '.join(query),
+            q, nocksum = find_checksum_id(' '.join(query),
                 distrib=distrib,
                 replica=replica,
                 latest=latest,

clef/esgf.py

@@ -47,7 +47,7 @@ class ESGFException(ClefException):
     pass
 
 
-def esgf_query(query=None, fields=[], otype='File', limit=20000, offset=0,  distrib=True, replica=False, latest=None,  **kwargs):
+def esgf_query(query=None, fields=[], otype='File', limit=10000, offset=0,  distrib=True, replica=False, latest=None,  **kwargs):
     """Search the ESGF
 
     Searches the ESGF using its `API
@@ -79,16 +79,16 @@ def esgf_query(query=None, fields=[], otype='File', limit=20000, offset=0,  dist
           'query': query,
           'fields': fields,
           'offset': offset,
-          #'limit': 20000,
+          'limit': limit,
           'distrib': distrib,
           'replica': replica,
           'latest': latest,
           'type': otype,
           'format': 'application/solr+json',
           }
     params.update(kwargs)
-    if otype == 'Dataset': params.pop('type')
-    print(f"Params: {params}")
+    if otype == 'Dataset':
+        params.pop('type')
 
     try:
         #r = requests.get('https://esgf-node.llnl.gov/esg-search/search',
@@ -189,6 +189,7 @@ def find_checksum_id(query, **kwargs):
                 records.append(doc)
             else:
                 nosums.append(doc)
+                print(doc)
 
     record_list = [ 
              (doc['checksum'][0],
@@ -215,7 +216,7 @@ def find_checksum_id(query, **kwargs):
             column('score', Float),
             name = 'esgf_table'
         ).data(record_list)
-    print(record_list)
+    nocksum = False
     if record_list == []:
         table = sqlalvalues(
             column('checksum', String),
@@ -226,9 +227,9 @@ def find_checksum_id(query, **kwargs):
             column('score', Float),
             name = 'esgf_table'
           ).data(nosums_list)
+        nocksum = True
 
-    print('this is ok')
-    return table
+    return table, nocksum
 
 
 def match_query(session, query, latest=None, **kwargs):
@@ -247,7 +248,7 @@ def match_query(session, query, latest=None, **kwargs):
     Returns:
         Joined result of :class:`clef.model.Path` and :func:`find_checksum_id`
     """
-    checksum_table = find_checksum_id(query, latest=latest, **kwargs)
+    checksum_table, nocksum = find_checksum_id(query, latest=latest, **kwargs)
 
     if latest is True:
         # Exact match on checksum
@@ -261,10 +262,13 @@ def match_query(session, query, latest=None, **kwargs):
         #return values.outerjoin(Path, Path.path.like('%/'+values.c.title))
         #return values.outerjoin(Path, func.regexp_replace(Path.path, '^.*/', '') == values.c.title)
         matches = checksum_table.join(Path, func.regexp_replace(Path.path, '^.*/', '') == checksum_table.c.title)
-    if checksum_table.c.checksum.contains("NA"):
-       matches = (checksum_table.join(C6Dataset, C6.Dataset.dataset_id == checksum_table.c.dataset_id)) 
+    
+    if nocksum is True:
+        if project == 'CMIP6':
+            matches = (checksum_table.join(C6Dataset, C6.Dataset.dataset_id == checksum_table.c.dataset_id)) 
     return matches
 
+
 def find_local_path(session, subq):
     """Find the filesystem paths of ESGF matches
 

docs/training3.rst

@@ -139,8 +139,8 @@ this is easy to implement and run. To make **search()** works for a
 random number of inputs passed by the command line we set up a function
 **call_local_query()** that deals with this more efficiently. The
 arguments are very similar to **search()** with the important difference
-that we are passing list of values instead of strings:
->call_local_query(s, project, oformat, latest, \**kwargs)
+that we are passing list of values instead of strings::
+    call_local_query(s, project, oformat, latest, \**kwargs)
 
 Let’s look at an example:
 

test/test_cli.py

@@ -293,7 +293,7 @@ def test_cite(runner, prod_cli):
     facets = ['--model=NESM3', '--experiment=historical', '--variant_label=r2i1p1f1',
               '--table=day', '--variable=tas', '--grid_label=gn', '--cite']
     # check that write_cite is called when cite passed with --local and --remote for CMIP6 
-    c = 'Cao, Jian; Wang, Bin (2019). NUIST NESMv3 model output prepared for CMIP6 CMIP historical. Version v20190812. Earth System Grid Federation. https://doi.org/10.22033/ESGF/CMIP6.8769'
+    c = 'Cao, Jian; Wang, Bin (2019). NUIST NESMv3 model output prepared for CMIP6 CMIP historical. Earth System Grid Federation. doi:https://doi.org/10.22033/ESGF/CMIP6.8769 .'
     with mock.patch('clef.cli.write_cite') as write_cite:
         r = prod_cli(['--local', 'cmip6', *facets])
         write_cite.assert_called_with([c])

test/test_code.py

@@ -155,7 +155,8 @@ def test_matching(session):
     }
     # Errors should print a message and return 'None'
     r = matching(session, ['variable','experiment'],['model','ensemble'], **facets)
-    assert r is None
+    assert r[0] is None
+    assert r[1] is None
 
 
 def test_ids_df(dids6, results6, dids5, results5):