Merge pull request #33 from cehbrecht/fix-tests

fixed tests

Author: agstephens

.travis.yml

@@ -1,17 +1,16 @@
+dist: xenial   # required for Python >= 3.7
 language: python
 python:
     - 2.7
-    - 3.3
-    - 3.4
     - 3.5
-
-addons:
-  apt:
-    packages:
-      - cython
+    - 3.6
+    - 3.7
+os:
+  - linux
+sudo: false
 install:
     - pip install -e .[testing]
 
 script:
-    - py.test
+    - pytest
     - flake8

README.rst

@@ -1,3 +1,15 @@
+.. image:: https://img.shields.io/badge/docs-latest-brightgreen.svg
+   :target: https://esgf-pyclient.readthedocs.io/en/latest/?badge=latest
+   :alt: Documentation Status
+
+.. image:: https://travis-ci.org/ESGF/esgf-pyclient.svg?branch=master
+   :target: https://travis-ci.org/ESGF/esgf-pyclient
+   :alt: Travis Build
+
+.. image:: https://img.shields.io/github/license/ESGF/esgf-pyclient.svg
+   :target: https://github.com/ESGF/esgf-pyclient/blob/master/LICENSE
+   :alt: GitHub license
+
 This package contains API code for calling the `ESGF Search API`_ within
 client code.  The initial implementation is in Python.
 

pyesgf/multidict.py

@@ -362,6 +362,8 @@ def __nonzero__(self):
                 return True
         return False
 
+    __bool__ = __nonzero__
+
     def iteritems(self):
         for d in self.dicts:
             for item in iteritems_(d):

pyesgf/search/connection.py

@@ -93,14 +93,13 @@ def __init__(self, url, distrib=True, cache=None, timeout=120,
         self._isopen = False
 
     def open(self):
-        if (isinstance(self._passed_session, requests.Session) or
-            isinstance(self._passed_session,
-                       requests_cache.core.CachedSession)):
+        if (isinstance(self._passed_session, requests.Session) or isinstance(
+                self._passed_session, requests_cache.core.CachedSession)):
             self.session = self._passed_session
         else:
             self.session = create_single_session(
-                                        cache=self.cache,
-                                        expire_after=self.expire_after)
+                cache=self.cache,
+                expire_after=self.expire_after)
         self._isopen = True
         return
 
@@ -114,9 +113,8 @@ def __exit__(self, type, value, traceback):
 
     def close(self):
         # Close the session
-        if not (isinstance(self._passed_session, requests.Session) or
-                isinstance(self._passed_session,
-                           requests_cache.core.CachedSession)):
+        if not (isinstance(self._passed_session, requests.Session) or isinstance(
+                self._passed_session, requests_cache.core.CachedSession)):
             self.session.close()
         self._isopen = False
         return
@@ -202,7 +200,7 @@ def _send_query(self, endpoint, full_query):
                                     timeout=self.timeout)
         if response.status_code == 400:
             # If error code 400, use urllib to find the errors:
-            errors = set(re.findall("Invalid HTTP query parameter=(\w+)",
+            errors = set(re.findall(r"Invalid HTTP query parameter=(\w+)",
                          response.text))
             content = "; ".join([e for e in list(errors)])
             raise Exception("Invalid query parameter(s): %s" % content)

pyesgf/test/test_ats.py

@@ -8,14 +8,16 @@
 from pyesgf.security.ats import AttributeService
 from pyesgf.node import ESGFNode
 from unittest import TestCase
+import pytest
 
 ESGF_NODE = ESGFNode('https://esgf-node.llnl.gov')
 TEST_OPENID = os.environ.get('LLNL_OPENID')
-TEST_USER_DETAILS = os.environ.get('LLNL_NAME').split()
+TEST_USER_DETAILS = os.environ.get('LLNL_NAME', '').split()
 
 
 class TestATS(TestCase):
 
+    @pytest.mark.skip(reason="needs test openid")
     def test_ceda_ats(self):
         service = AttributeService(ESGF_NODE.ats_url, 'esgf-pyclient')
         fn, ln = TEST_USER_DETAILS
@@ -37,6 +39,7 @@ def test_unknown_principal(self):
         assert resp.get_status() == ('urn:oasis:names:tc:SAML:2.0:'
                                      'status:UnknownPrincipal')
 
+    @pytest.mark.skip(reason="needs test openid")
     def test_multi_attribute(self):
         service = AttributeService(ESGF_NODE.ats_url, 'esgf-pyclient')
         CMIP5_RESEARCH = 'CMIP5 Research'

pyesgf/test/test_context.py

@@ -10,7 +10,7 @@
 
 class TestContext(TestCase):
     def setUp(self):
-        self.test_service = 'http://esgf-index1.ceda.ac.uk/esg-search'
+        self.test_service = 'http://esgf-data.dkrz.de/esg-search'
         self.cache = os.path.join(os.path.dirname(__file__), 'url_cache')
 
     def test_context_freetext(self):
@@ -35,8 +35,7 @@ def test_context_facets_multivalue(self):
         self.assertTrue(context2.hit_count > 0)
 
         self.assertTrue(context2.facet_constraints['project'] == 'CMIP5')
-        self.assertTrue(sorted(context2.facet_constraints.getall('model'))
-                        == ['IPSL-CM5A-LR', 'IPSL-CM5A-MR'])
+        self.assertTrue(sorted(context2.facet_constraints.getall('model')) == ['IPSL-CM5A-LR', 'IPSL-CM5A-MR'])
 
     def test_context_facet_multivalue2(self):
         conn = SearchConnection(self.test_service, cache=self.cache)
@@ -45,8 +44,7 @@ def test_context_facet_multivalue2(self):
             context.facet_constraints.getall('model') == ['IPSL-CM5A-MR'])
 
         context2 = context.constrain(model=['IPSL-CM5A-MR', 'IPSL-CM5A-LR'])
-        self.assertTrue(sorted(context2.facet_constraints.getall('model'))
-                        == ['IPSL-CM5A-LR', 'IPSL-CM5A-MR'])
+        self.assertTrue(sorted(context2.facet_constraints.getall('model')) == ['IPSL-CM5A-LR', 'IPSL-CM5A-MR'])
 
     def test_context_facet_multivalue3(self):
         conn = SearchConnection(self.test_service, cache=self.cache)
@@ -94,18 +92,29 @@ def test_facet_count(self):
         self.assertTrue(list(counts['project'].keys()) == ['CMIP5'])
 
     def test_distrib(self):
-        conn = SearchConnection(self.test_service, cache=self.cache,
-                                distrib=False)
+        conn = SearchConnection(self.test_service, distrib=False)
 
         context = conn.new_context(project='CMIP5')
         count1 = context.hit_count
 
-        conn2 = SearchConnection(self.test_service, cache=self.cache,
-                                 distrib=True)
+        conn2 = SearchConnection(self.test_service, distrib=True)
         context = conn2.new_context(project='CMIP5')
         count2 = context.hit_count
 
-        self.assertTrue(count1 < count2)
+        assert count1 < count2
+
+    # @pytest.mark.skip(reason="cache fails on python 3.7")
+    def test_distrib_with_cache(self):
+        conn = SearchConnection(self.test_service, cache=self.cache, distrib=False)
+
+        context = conn.new_context(project='CMIP5')
+        count1 = context.hit_count
+
+        conn2 = SearchConnection(self.test_service, cache=self.cache, distrib=True)
+        context = conn2.new_context(project='CMIP5')
+        count2 = context.hit_count
+
+        assert count1 < count2
 
     def test_constrain(self):
         conn = SearchConnection(self.test_service, cache=self.cache)
@@ -149,24 +158,24 @@ def test_negative_facet(self):
         context3 = context.constrain(experiment=not_equals('historical'))
         hits3 = context3.hit_count
 
-        self.assertTrue(hits1 == hits2 + hits3)
+        assert hits1 == hits2 + hits3
 
     def test_replica(self):
         # Test that we can exclude replicas
         # This tests assumes the test dataset is replicated
-        conn = SearchConnection(self.test_service, cache=self.cache)
+        conn = SearchConnection(self.test_service)
         qry = 'id:cmip5.output1.MOHC.HadGEM2-ES.rcp45.mon.atmos.Amon.r1i1p1.*'
         version = '20111128'
 
         # Search for all replicas
         context = conn.new_context(query=qry, version=version)
-        self.assertTrue(context.hit_count > 2,
-                        'Expecting more than 2 search hits for replicas')
+        # Expecting more than 2 search hits for replicas
+        assert context.hit_count > 2
 
         # Search for only one replicant
         context = conn.new_context(query=qry, replica=False, version=version)
-        self.assertTrue(context.hit_count == 1,
-                        'Expecting one search replica')
+        # Expecting one search replica
+        assert context.hit_count == 1
 
     def test_response_from_bad_parameter(self):
         # Test that a bad parameter name raises a useful exception
@@ -186,21 +195,18 @@ def test_response_from_bad_parameter(self):
                 "No JSON object could be decoded"))
 
     def test_context_project_cmip6(self):
-        test_service = 'https://esgf-node.ipsl.upmc.fr/esg-search'
+        test_service = 'https://esgf-node.llnl.gov/esg-search'
         conn = SearchConnection(test_service)
 
-        context = conn.new_context(project='CMIP6')
-        self.assertEqual(context.hit_count, 1973)
-
-        context2 = context.constrain(realm="atmosChem")
-        self.assertEqual(context2.hit_count, 10)
+        context = conn.new_context(project='CMIP6', institution_id='AWI', distrib=False)
+        self.assertTrue(context.hit_count > 100)
 
+        context2 = context.constrain(variable='tas')
+        self.assertTrue(context2.hit_count > 10)
 
     def test_context_project_c3s_cmip5(self):
         test_service = 'https://cp4cds-index1.ceda.ac.uk/esg-search'
         conn = SearchConnection(test_service)
 
         context = conn.new_context(project='c3s-cmip5')
         self.assertTrue(context.hit_count > 20000)
-
-

pyesgf/test/test_results.py

@@ -23,7 +23,7 @@ def test_result1(self):
         results = ctx.search()
 
         r1 = results[0]
-        assert re.match(r'cmip5\.output1\.MOHC\..+\|esgf-data1.ceda.ac.uk',
+        assert re.match(r'cmip5\.output1\..+\|esgf-data1.ceda.ac.uk',
                         r1.dataset_id)
 
     def test_file_context(self):

pyesgf/test/test_shard_regex.py

@@ -5,8 +5,8 @@
 import re
 from unittest import TestCase
 
-R = re.compile("^(?P<prefix>https?://)?(?P<host>.+?)"
-               ":?(?P<port>\d+)?/(?P<suffix>.+)$")
+R = re.compile(r"^(?P<prefix>https?://)?(?P<host>.+?)"
+               r":?(?P<port>\d+)?/(?P<suffix>.+)$")
 
 
 class TestShardRegex(TestCase):

setup.cfg

@@ -1,15 +1,15 @@
-[egg_info]
-#tag_build = p3
-
-[nosetests]
-verbosity=2
-with-doctest=1
-doctest-extension=.txt
-tests=test
-#nologcapture=1
+[tool:pytest]
+addopts =
+	--strict
+	--tb=native
+python_files = test_*.py
+markers =
+	online: mark test to need internet connection
+	slow: mark test to be slow
 
 [flake8]
-exclude = 
+max-line-length = 120
+exclude =
     .git,
     __pycache__,
     docs/conf.py,

setup.py

@@ -19,16 +19,14 @@
       description="A library interacting with ESGF services within Python",
       long_description=long_description,
       classifiers=[
-        'Development Status :: 4 - Beta',
-        'License :: OSI Approved :: BSD License',
-        'Topic :: Scientific/Engineering',
-        'Programming Language :: Python :: 2.6',
-        'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.3',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6'
-        ],
+          'Development Status :: 4 - Beta',
+          'License :: OSI Approved :: BSD License',
+          'Topic :: Scientific/Engineering',
+          'Programming Language :: Python :: 2.7',
+          'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
+          'Programming Language :: Python :: 3.7',
+      ],
       # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
       keywords='',
       author='Ag Stephens',