Refactor independent cluster configs, pin pandas<2 in cluster environments (#1120)

* Pin independent cluster pandas to <2

* Refactor cluster config files to make environments a little more readable

Author: charlesbluca

.github/cluster-upstream.yml

@@ -129,9 +129,9 @@ jobs:
       - name: run a dask cluster
         run: |
           if [[ $which_upstream == "Dask" ]]; then
-            docker-compose -f .github/cluster-upstream.yml up -d
+            docker-compose -f continuous_integration/cluster/upstream.yml up -d
           else
-            docker-compose -f .github/cluster.yml up -d
+            docker-compose -f continuous_integration/cluster/stable.yml up -d
           fi
 
           # periodically ping logs until a connection has been established; assume failure after 2 minutes

.github/workflows/test-upstream.yml

@@ -125,9 +125,9 @@ jobs:
           UPSTREAM: ${{ needs.detect-ci-trigger.outputs.triggered }}
         run: |
           if [[ $UPSTREAM == "true" ]]; then
-            docker-compose -f .github/cluster-upstream.yml up -d
+            docker-compose -f continuous_integration/cluster/upstream.yml up -d
           else
-            docker-compose -f .github/cluster.yml up -d
+            docker-compose -f continuous_integration/cluster/stable.yml up -d
           fi
 
           # periodically ping logs until a connection has been established; assume failure after 2 minutes

.github/workflows/test.yml

@@ -0,0 +1,7 @@
+name: base
+channels:
+- conda-forge
+- nodefaults
+dependencies:
+# dask serialization needs core libraries to be consistent on client/cluster
+- pandas>=1.4.0,<2

continuous_integration/cluster/environment.yml

@@ -4,20 +4,19 @@ services:
     dask-scheduler:
         container_name: dask-scheduler
         image: daskdev/dask:dev-py3.9
-        command: dask-scheduler
+        command: dask scheduler
         ports:
             - "8786:8786"
         environment:
             USE_MAMBA: "true"
-            # p2p shuffling requires pyarrow>=7.0.0
-            EXTRA_CONDA_PACKAGES: "pyarrow>=7.0.0"
+        volumes:
+            - ./environment.yml:/opt/app/environment.yml
     dask-worker:
         container_name: dask-worker
         image: daskdev/dask:dev-py3.9
-        command: dask-worker dask-scheduler:8786
+        command: dask worker dask-scheduler:8786
         environment:
             USE_MAMBA: "true"
-            # TODO: remove pandas constraint once Dask images are updated
-            EXTRA_CONDA_PACKAGES: "cloudpickle>=2.1.0 pyarrow>=6.0.1 libstdcxx-ng>=12.1.0 pandas>=1.5.0"
         volumes:
+            - ./environment.yml:/opt/app/environment.yml
             - /tmp:/tmp

.github/cluster.yml renamed to continuous_integration/cluster/stable.yml

@@ -0,0 +1,24 @@
+# Docker-compose setup used during tests
+version: '3'
+services:
+    dask-scheduler:
+        container_name: dask-scheduler
+        image: daskdev/dask:dev-py3.9
+        command: dask scheduler
+        ports:
+            - "8786:8786"
+        environment:
+            USE_MAMBA: "true"
+            EXTRA_CONDA_PACKAGES: "dask/label/dev::dask"
+        volumes:
+            - ./environment.yml:/opt/app/environment.yml
+    dask-worker:
+        container_name: dask-worker
+        image: daskdev/dask:dev-py3.9
+        command: dask worker dask-scheduler:8786
+        environment:
+            USE_MAMBA: "true"
+            EXTRA_CONDA_PACKAGES: "dask/label/dev::dask"
+        volumes:
+            - ./environment.yml:/opt/app/environment.yml
+            - /tmp:/tmp