Merge pull request #35 from andersy005/master

Code Refactoring : Allow CMIP integration

Author: matt-long

.circleci/config.yml

@@ -6,13 +6,14 @@ workflows:
     jobs:
       - "python-3.7"
       - "python-3.6"
-      - "docs-and-linting"
+      - "docs-build"
+      - "test-linting"
+
 
-        
 jobs:
   "python-3.6":
     docker:
-      - image: continuumio/miniconda:latest
+      - image: conda/miniconda3:latest
 
     environment:
       PYTHON: "3.6"
@@ -26,7 +27,7 @@ jobs:
           key: deps-{{ .Branch }}-3.6-{{ checksum "ci/environment-dev-3.6.yml" }}
 
       - run: # install and activate conda environment
-          name: configure conda 
+          name: Install conda environment
           command: ./ci/install-circle.sh
 
       - run:
@@ -36,7 +37,7 @@ jobs:
 
 
       - run:
-          name: Code Coverage 
+          name: Code Coverage
           command: |
             source activate ${ENV_NAME}
 
@@ -57,14 +58,14 @@ jobs:
 
   "python-3.7":
     docker:
-      - image: continuumio/miniconda:latest
+      - image: conda/miniconda3:latest
 
     environment:
       PYTHON: "3.7"
       ENV_NAME: "intake-esm-dev"
 
     steps:
-    
+
       # Get our data and merge with upstream
       - checkout
 
@@ -73,22 +74,22 @@ jobs:
           key: deps-{{ .Branch }}-3.7-{{ checksum "ci/environment-dev-3.7.yml" }}
 
       - run: # install and activate conda environment
-          name: configure conda 
+          name: Install conda environment
           command: ./ci/install-circle.sh
       - run:
           command: |
             source activate ${ENV_NAME}
             pytest --junitxml=test-reports/junit.xml --cov=./ --verbose
 
       - run:
-          name: Code Coverage 
+          name: Code Coverage
           command: |
             source activate ${ENV_NAME}
 
             echo "[Upload coverage]"
             codecov
 
-     
+
       # Cache some files for a speedup in subsequent builds
       - save_cache:
           key: deps-{{ .Branch }}-3.7-{{ checksum "ci/environment-dev-3.7.yml" }}
@@ -101,19 +102,19 @@ jobs:
 
       - store_artifacts:
           path: test-reports
-   
-  
 
-  "docs-and-linting":
+
+
+  "test-linting":
     docker:
-      - image: continuumio/miniconda:latest
+      - image: conda/miniconda3:latest
 
     environment:
       PYTHON: "3.7"
       ENV_NAME: "intake-esm-dev"
 
     steps:
-    
+
       # Get our data and merge with upstream
       - checkout
 
@@ -122,32 +123,62 @@ jobs:
           key: deps-{{ .Branch }}-3.7-{{ checksum "ci/environment-dev-3.7.yml" }}
 
       - run: # install and activate conda environment
-          name: configure conda 
+          name: Install conda environment
           command: ./ci/install-circle.sh
-     
+
 
       - run:
           name: Code Styling with (black, flake8, isort)
           command: |
             source activate ${ENV_NAME}
-            
+
             echo "[flake8]"
-            flake8 
+            flake8
 
             echo "[black]"
-            black --check . 
+            black --check --line-length=100 -S --exclude='intake_esm/_version.py|versioneer.py' .
 
             echo "[isort]"
-            isort --recursive --check-only .
+            isort --recursive -w 100 --check-only .
+
+      # Cache some files for a speedup in subsequent builds
+      - save_cache:
+          key: deps-{{ .Branch }}-3.7-{{ checksum "ci/environment-dev-3.7.yml" }}
+          paths:
+            - "/opt/conda/envs/${ENV_NAME}/"
+            - "/opt/conda/pkgs"
+
+
+  "docs-build":
+    docker:
+      - image: conda/miniconda3:latest
+
+    environment:
+      PYTHON: "3.7"
+      ENV_NAME: "intake-esm-dev"
+
+    steps:
+
+      # Get our data and merge with upstream
+      - checkout
+
+      # Restore cached files to speed things up
+      - restore_cache:
+          key: deps-{{ .Branch }}-3.7-{{ checksum "ci/environment-dev-3.7.yml" }}
+
+      - run: # install and activate conda environment
+          name: Install conda environment
+          command: ./ci/install-circle.sh
+
       - run:
-          name: Check documentation build 
+          name: Check documentation build
           command: |
             source activate ${ENV_NAME}
             pushd docs
             make html
             popd
 
-     
+
       # Cache some files for a speedup in subsequent builds
       - save_cache:
           key: deps-{{ .Branch }}-3.7-{{ checksum "ci/environment-dev-3.7.yml" }}
@@ -156,6 +187,6 @@ jobs:
             - "/opt/conda/pkgs"
 
       # Tell Circle to store the documentation output in a folder that we can access later
-      - store_artifacts: 
+      - store_artifacts:
           path: docs/_build/html
-          destination: html
+          destination: html

.pre-commit-config.yaml

@@ -1,20 +1,34 @@
 repos:
+
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.1.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-docstring-first
+    -   id: check-json
+    -   id: check-added-large-files
+    -   id: check-yaml
+    -   id: double-quote-string-fixer
+
 -   repo: https://github.com/ambv/black
     rev: 18.9b0
     hooks:
     - id: black
-
+      args: ["--line-length", "100", "--skip-string-normalization",
+            "--exclude", '''/(intake_esm/_version.py|versioneer.py)/''']
 
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.1.0
     hooks:
     - id: flake8
 
 -   repo: https://github.com/asottile/seed-isort-config
-    rev: v1.5.0
+    rev: v1.6.0
     hooks:
     -   id: seed-isort-config
 -   repo: https://github.com/pre-commit/mirrors-isort
     rev: v4.3.4
     hooks:
-    -   id: isort
+    -   id: isort
+        args: ["line_length=100"]

README.rst

@@ -22,7 +22,7 @@ Intake-esm
     :alt: Conda Version
 
 
-Intake-esm provides a plug for reading esm Large Ensemble data sets using intake.
+Intake-esm provides a plugin for building and loading intake catalogs for earth system data sets holdings, such as CMIP and CESM Large Ensemble datasets. 
 See documentation_ for more information.
 
 .. _documentation: https://intake-esm.readthedocs.io/en/latest/

ci/environment-dev-3.6.yml

@@ -20,4 +20,4 @@ dependencies:
   - pyyaml
   - tqdm
   - xarray
-  - pip=18.1
+  - pip

ci/environment-dev-3.7.yml

@@ -30,7 +30,7 @@ dependencies:
   - sphinx>=1.6
   - tqdm
   - xarray
-  - pip=18.1
+  - pip
   - make
   - pip:
     - sphinx_copybutton

ci/install-circle.sh

@@ -4,7 +4,6 @@ set -e
 set -eo pipefail
 
 conda config --set always_yes true --set changeps1 false --set quiet true
-conda update -q conda
 conda config --add channels conda-forge
 conda env create -f ci/environment-dev-${PYTHON}.yml --name=${ENV_NAME} --quiet
 conda env list

docs/source/api.rst

@@ -6,27 +6,14 @@ This is a reference API class listing, and modules.
 .. currentmodule:: intake_esm
 
 .. autosummary::
-   core.CesmMetadataStoreCatalog
-   core.CesmSource
-   manage_collections.StorageResource
-   manage_collections.CESMCollections
-  
+   core.ESMMetadataStoreCatalog
 
-.. currentmodule:: intake_esm.core
-
-.. autoclass:: CesmMetadataStoreCatalog
-   :members: __init__, search 
-
-.. autoclass:: CesmSource
-   :members: __init__, to_xarray
 
+.. currentmodule:: intake_esm.core
 
-.. currentmodule:: intake_esm.manage_collections
+.. autoclass:: ESMMetadataStoreCatalog
+   :members: __init__, search, build_collections, open_collection, get_built_collections 
 
-.. autoclass:: StorageResource
-   :members: __init__
 
-.. autoclass:: CESMCollections
-   :members: __init__, get_built_collection