Skip to content

Commit 3839df1

Browse files
glemaitreglemaitre
authored
MNT activate azure pipelines (#652)
1 parent 06fe9ba commit 3839df1

File tree

16 files changed

+705
-34
lines changed

16 files changed

+705
-34
lines changed

‎README.rst

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@
44

55
.. _scikit-learn-contrib: https://github.com/scikit-learn-contrib
66

7-
|Travis|_ |AppVeyor|_ |Codecov|_ |CircleCI|_ |PythonVersion|_ |Pypi|_ |Gitter|_
7+
|Azure|_ |Travis|_ |AppVeyor|_ |Codecov|_ |CircleCI|_ |PythonVersion|_ |Pypi|_ |Gitter|_
8+
9+
.. |Azure| image:: https://dev.azure.com/imbalanced-learn/imbalanced-learn/_apis/build/status/scikit-learn-contrib.imbalanced-learn?branchName=master
10+
.. _Azure: https://dev.azure.com/imbalanced-learn/imbalanced-learn/_build
811

912
.. |Travis| image:: https://travis-ci.org/scikit-learn-contrib/imbalanced-learn.svg?branch=master
1013
.. _Travis: https://travis-ci.org/scikit-learn-contrib/imbalanced-learn

‎azure-pipelines.yml

Lines changed: 101 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,105 @@
1-
# Python package
2-
# Create and test a Python package on multiple Python versions.
3-
# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
4-
# https://docs.microsoft.com/azure/devops/pipelines/languages/python
1+
# Adapted from https://github.com/scikit-learn/scikit-learn/blob/master/azure-pipelines.yml
2+
jobs:
3+
- job: linting
4+
displayName: Linting
5+
pool:
6+
vmImage: ubuntu-16.04
7+
steps:
8+
- bash: echo "##vso[task.prependpath]$CONDA/bin"
9+
displayName: Add conda to PATH
10+
- bash: sudo chown -R $USER $CONDA
11+
displayName: Take ownership of conda installation
12+
- bash: conda create --name flake8_env --yes flake8
13+
displayName: Install flake8
14+
- bash: |
15+
source activate flake8_env
16+
./build_tools/circle/linting.sh
17+
displayName: Run linting
518
6-
trigger:
7-
- master
19+
- template: build_tools/azure/posix.yml
20+
parameters:
21+
name: Linux
22+
vmImage: ubuntu-16.04
23+
dependsOn: [linting]
24+
matrix:
25+
# Linux environment to test that scikit-learn can be built against
26+
# versions of numpy, scipy with ATLAS that comes with Ubuntu Xenial 16.04
27+
# i.e. numpy 1.11 and scipy 0.17
28+
py35_ubuntu_atlas:
29+
DISTRIB: 'ubuntu'
30+
PYTHON_VERSION: '3.5'
31+
JOBLIB_VERSION: '*'
32+
# Linux environment to test the latest available dependencies and MKL.
33+
pylatest_pip_openblas_pandas:
34+
DISTRIB: 'conda-pip-latest'
35+
PYTHON_VERSION: '3.7'
36+
COVERAGE: 'true'
37+
PANDAS_VERSION: '*'
38+
TEST_DOCSTRINGS: 'true'
39+
JOBLIB_VERSION: '*'
40+
CHECK_WARNINGS: 'true'
41+
pylatest_conda_pandas_keras:
42+
DISTRIB: 'conda'
43+
PYTHON_VERSION: '3.7'
44+
INSTALL_MKL: 'true'
45+
PANDAS_VERSION: '*'
46+
KERAS_VERSION: '*'
47+
COVERAGE: 'true'
48+
JOBLIB_VERSION: '*'
49+
TEST_DOCSTRINGS: 'true'
50+
pylatest_conda_pandas_tensorflow:
51+
DISTRIB: 'conda'
52+
PYTHON_VERSION: '3.7'
53+
PANDAS_VERSION: '*'
54+
JOBLIB_VERSION: '*'
55+
INSTALL_MKL: 'true'
56+
TENSORFLOW_VERSION: '*'
57+
COVERAGE: 'true'
58+
TEST_DOCSTRINGS: 'true'
859

9-
pool:
10-
vmImage: 'ubuntu-latest'
11-
strategy:
12-
matrix:
13-
Python27:
14-
python.version: '2.7'
15-
Python35:
16-
python.version: '3.5'
17-
Python36:
18-
python.version: '3.6'
19-
Python37:
20-
python.version: '3.7'
60+
- template: build_tools/azure/posix-32.yml
61+
parameters:
62+
name: Linux32
63+
vmImage: ubuntu-16.04
64+
dependsOn: [linting]
65+
matrix:
66+
py35_ubuntu_atlas_32bit:
67+
DISTRIB: 'ubuntu-32'
68+
PYTHON_VERSION: '3.5'
69+
JOBLIB_VERSION: '*'
70+
TEST_DOCSTRINGS: 'true'
2171

22-
steps:
23-
- task: UsePythonVersion@0
24-
inputs:
25-
versionSpec: '$(python.version)'
26-
displayName: 'Use Python $(python.version)'
72+
- template: build_tools/azure/posix.yml
73+
parameters:
74+
name: macOS
75+
vmImage: xcode9-macos10.13
76+
dependsOn: [linting]
77+
matrix:
78+
pylatest_conda_mkl:
79+
DISTRIB: 'conda'
80+
PYTHON_VERSION: '*'
81+
INSTALL_MKL: 'true'
82+
NUMPY_VERSION: '*'
83+
SCIPY_VERSION: '*'
84+
PANDAS_VERSION: '*'
85+
PYTEST_VERSION: '*'
86+
JOBLIB_VERSION: '*'
87+
COVERAGE: 'true'
88+
TEST_DOCSTRINGS: 'true'
89+
CHECK_WARNINGS: 'true'
2790

28-
- script: |
29-
python -m pip install --upgrade pip
30-
pip install -r requirements.txt
31-
displayName: 'Install dependencies'
32-
33-
- script: |
34-
pip install pytest pytest-azurepipelines
35-
pytest
36-
displayName: 'pytest'
91+
- template: build_tools/azure/windows.yml
92+
parameters:
93+
name: Windows
94+
vmImage: vs2017-win2016
95+
dependsOn: [linting]
96+
matrix:
97+
py37_conda_mkl:
98+
PYTHON_VERSION: '3.7'
99+
PYTHON_ARCH: '64'
100+
PYTEST_VERSION: '*'
101+
COVERAGE: 'true'
102+
CHECK_WARNINGS: 'true'
103+
py35_pip_openblas_32bit:
104+
PYTHON_VERSION: '3.5'
105+
PYTHON_ARCH: '32'

‎build_tools/azure/install.cmd

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
@rem https://github.com/numba/numba/blob/master/buildscripts/incremental/setup_conda_environment.cmd
2+
@rem The cmd /C hack circumvents a regression where conda installs a conda.bat
3+
@rem script in non-root environments.
4+
set CONDA_INSTALL=cmd /C conda install -q -y
5+
set PIP_INSTALL=pip install -q
6+
7+
@echo on
8+
9+
IF "%PYTHON_ARCH%"=="64" (
10+
@rem Deactivate any environment
11+
call deactivate
12+
@rem Clean up any left-over from a previous build
13+
conda remove --all -q -y -n %VIRTUALENV%
14+
conda create -n %VIRTUALENV% -q -y python=%PYTHON_VERSION% numpy scipy cython wheel joblib git
15+
16+
call activate %VIRTUALENV%
17+
18+
IF "%PYTEST_VERSION%"=="*" (
19+
pip install pytest
20+
) else (
21+
pip install pytest==%PYTEST_VERSION%
22+
)
23+
pip install pytest-xdist
24+
) else (
25+
pip install numpy scipy cython pytest wheel pillow joblib
26+
)
27+
if "%COVERAGE%" == "true" (
28+
pip install coverage codecov pytest-cov
29+
)
30+
python --version
31+
pip --version
32+
33+
pip install git+https://github.com/scikit-learn/scikit-learn.git
34+
35+
@rem Install the build and runtime dependencies of the project.
36+
python setup.py bdist_wheel bdist_wininst
37+
38+
@rem Install the generated wheel package to test it
39+
pip install --pre --no-index --find-links dist\ imbalanced-learn
40+
41+
if %errorlevel% neq 0 exit /b %errorlevel%

‎build_tools/azure/install.sh

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
UNAMESTR=`uname`
6+
7+
make_conda() {
8+
TO_INSTALL="$@"
9+
conda create -n $VIRTUALENV --yes $TO_INSTALL
10+
source activate $VIRTUALENV
11+
}
12+
13+
version_ge() {
14+
# The two version numbers are seperated with a new line is piped to sort
15+
# -rV. The -V activates for version number sorting and -r sorts in
16+
# decending order. If the first argument is the top element of the sort, it
17+
# is greater than or equal to the second argument.
18+
test "$(printf "${1}\n${2}" | sort -rV | head -n 1)" == "$1"
19+
}
20+
21+
if [[ "$DISTRIB" == "conda" ]]; then
22+
23+
TO_INSTALL="python=$PYTHON_VERSION pip \
24+
numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION \
25+
joblib=$JOBLIB_VERSION git"
26+
27+
if [[ "$INSTALL_MKL" == "true" ]]; then
28+
TO_INSTALL="$TO_INSTALL mkl"
29+
else
30+
TO_INSTALL="$TO_INSTALL nomkl"
31+
fi
32+
33+
make_conda $TO_INSTALL
34+
python -m pip install --pre -f https://sklearn-nightly.scdn8.secure.raxcdn.com scikit-learn
35+
36+
TO_INSTALL=""
37+
38+
if [[ -n "$PANDAS_VERSION" ]]; then
39+
TO_INSTALL="$TO_INSTALL pandas=$PANDAS_VERSION"
40+
fi
41+
42+
if [[ -n "$KERAS_VERSION" ]]; then
43+
TO_INSTALL="$TO_INSTALL keras=$KERAS_VERSION tensorflow=1"
44+
KERAS_BACKEND=tensorflow
45+
fi
46+
47+
if [[ -n "$TENSORFLOW_VERSION" ]]; then
48+
TO_INSTALL="$TO_INSTALL tensorflow=$TENSORFLOW_VERSION"
49+
fi
50+
51+
if [[ "$PYTEST_VERSION" == "*" ]]; then
52+
python -m pip install pytest
53+
else
54+
python -m pip install pytest=="$PYTEST_VERSION"
55+
fi
56+
57+
if [[ "$PYTHON_VERSION" == "*" ]]; then
58+
python -m pip install pytest-xdist
59+
fi
60+
61+
if [[ -n "$TO_INSTALL" ]]; then
62+
conda install --yes $TO_INSTALL
63+
fi
64+
65+
if [[ -n "$KERAS_VERSION" ]]; then
66+
python -c "import keras.backend"
67+
sed -i -e 's/"backend":[[:space:]]*"[^"]*/"backend":\ "'$KERAS_BACKEND'/g' ~/.keras/keras.json;
68+
fi
69+
70+
elif [[ "$DISTRIB" == "ubuntu" ]]; then
71+
sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test
72+
sudo apt-get update
73+
sudo apt-get install python3-scipy libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv git
74+
python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV
75+
source $VIRTUALENV/bin/activate
76+
python -m pip install pandas
77+
python -m pip install pytest==$PYTEST_VERSION pytest-cov joblib cython
78+
python -m pip install git+https://github.com/scikit-learn/scikit-learn.git
79+
elif [[ "$DISTRIB" == "ubuntu-32" ]]; then
80+
apt-get update
81+
apt-get install -y python3-dev python3-scipy libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv git
82+
python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV
83+
source $VIRTUALENV/bin/activate
84+
python -m pip install pandas
85+
python -m pip install pytest==$PYTEST_VERSION pytest-cov joblib cython
86+
python -m pip install git+https://github.com/scikit-learn/scikit-learn.git
87+
elif [[ "$DISTRIB" == "conda-pip-latest" ]]; then
88+
# Since conda main channel usually lacks behind on the latest releases,
89+
# we use pypi to test against the latest releases of the dependencies.
90+
# conda is still used as a convenient way to install Python and pip.
91+
make_conda "python=$PYTHON_VERSION"
92+
python -m pip install -U pip
93+
python -m pip install numpy scipy joblib cython
94+
python -m pip install git+https://github.com/scikit-learn/scikit-learn.git
95+
python -m pip install pytest==$PYTEST_VERSION pytest-cov pytest-xdist
96+
python -m pip install pandas
97+
fi
98+
99+
if [[ "$COVERAGE" == "true" ]]; then
100+
python -m pip install coverage codecov pytest-cov
101+
fi
102+
103+
if [[ "$TEST_DOCSTRINGS" == "true" ]]; then
104+
python -m pip install sphinx
105+
python -m pip install -U git+https://github.com/numpy/numpydoc.git
106+
fi
107+
108+
python --version
109+
python -c "import numpy; print('numpy %s' % numpy.__version__)"
110+
python -c "import scipy; print('scipy %s' % scipy.__version__)"
111+
python -c "\
112+
try:
113+
import pandas
114+
print('pandas %s' % pandas.__version__)
115+
except ImportError:
116+
print('pandas not installed')
117+
"
118+
python -m pip list
119+
120+
# Use setup.py instead of `pip install -e .` to be able to pass the -j flag
121+
# to speed-up the building multicore CI machines.
122+
python setup.py develop

‎build_tools/azure/posix-32.yml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
parameters:
2+
name: ''
3+
vmImage: ''
4+
matrix: []
5+
dependsOn: []
6+
7+
jobs:
8+
- job: ${{ parameters.name }}
9+
dependsOn: ${{ parameters.dependsOn }}
10+
pool:
11+
vmImage: ${{ parameters.vmImage }}
12+
variables:
13+
TEST_DIR: '$(Agent.WorkFolder)/tmp_folder'
14+
JUNITXML: 'test-data.xml'
15+
OMP_NUM_THREADS: '4'
16+
PYTEST_VERSION: '5.2.1'
17+
OPENBLAS_NUM_THREADS: '4'
18+
SKLEARN_SKIP_NETWORK_TESTS: '1'
19+
strategy:
20+
matrix:
21+
${{ insert }}: ${{ parameters.matrix }}
22+
23+
steps:
24+
# Container is detached and sleeping, allowing steps to run commmands
25+
# in the container. The TEST_DIR is mapped allowing the host to access
26+
# the JUNITXML file
27+
- script: >
28+
docker container run --rm
29+
--volume $TEST_DIR:/temp_dir
30+
--volume $PWD:/io
31+
-w /io
32+
--detach
33+
--name skcontainer
34+
-e DISTRIB=ubuntu-32
35+
-e TEST_DIR=/temp_dir
36+
-e JUNITXML=$JUNITXML
37+
-e VIRTUALENV=testvenv
38+
-e JOBLIB_VERSION=$JOBLIB_VERSION
39+
-e PYTEST_VERSION=$PYTEST_VERSION
40+
-e OMP_NUM_THREADS=$OMP_NUM_THREADS
41+
-e OPENBLAS_NUM_THREADS=$OPENBLAS_NUM_THREADS
42+
-e SKLEARN_SKIP_NETWORK_TESTS=$SKLEARN_SKIP_NETWORK_TESTS
43+
i386/ubuntu:16.04
44+
sleep 1000000
45+
displayName: 'Start container'
46+
- script: >
47+
docker exec skcontainer ./build_tools/azure/install.sh
48+
displayName: 'Install'
49+
- script: >
50+
docker exec skcontainer ./build_tools/azure/test_script.sh
51+
displayName: 'Test Library'
52+
- task: PublishTestResults@2
53+
inputs:
54+
testResultsFiles: '$(TEST_DIR)/$(JUNITXML)'
55+
testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }}
56+
displayName: 'Publish Test Results'
57+
condition: succeededOrFailed()
58+
- script: >
59+
docker container stop skcontainer
60+
displayName: 'Stop container'
61+
condition: always()

0 commit comments

Comments
 (0)