Merge pull request #50 from rapidsai/branch-22.04

[RELEASE] kvikio v22.04

.gitignore

@@ -0,0 +1,13 @@
+*.*~
+build
+python/_skbuild
+*.so
+dask-worker-space
+__pycache__
+.cache
+*.egg-info
+.vscode
+python/kvikio/_lib/*.cpp
+cufile.log
+docs/build/
+cpp/doxygen/html/

README.md

@@ -34,7 +34,13 @@ For testing:
 
 ## Install
 
-### C++
+### Conda 
+Install the `kvikio` conda package from the `rapidsai-nightly` channel like:
+```
+conda create -n kvikio_env -c rapidsai-nightly -c conda-forge python=3.8 cudatoolkit=11.5 kvikio
+```
+
+### C++ (build from source)
 To build the C++ example, go to the `cpp` subdiretory and run:
 ```
 mkdir build
@@ -47,7 +53,7 @@ Then run the example:
 ./examples/basic_io
 ```
 
-### Python
+### Python (build from source)
 
 To build and install the extension, go to the `python` subdiretory and run:
 ```
@@ -121,4 +127,19 @@ f = kvikio.CuFile("test-file", "r")
 # Read whole array from file
 f.read(b)
 assert all(a == b)
+
+# Use contexmanager
+c = cupy.empty_like(a)
+with kvikio.CuFile(path, "r") as f:
+    f.read(c)
+assert all(a == c)
+
+# Non-blocking read
+d = cupy.empty_like(a)
+with kvikio.CuFile(path, "r") as f:
+    future1 = f.pread(d[:50])
+    future2 = f.pread(d[50:], file_offset=d[:50].nbytes)
+    future1.get()  # Wait for first read
+    future2.get()  # Wait for second read
+assert all(a == d)
 ```

ci/checks/style.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#######################
+# kvikio Style Tester #
+#######################
+
+# Ignore errors and set path
+set +e
+PATH=/conda/bin:$PATH
+LC_ALL=C.UTF-8
+LANG=C.UTF-8
+
+# Activate common conda env
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
+
+# Run formatting script
+FORMAT=`python scripts/format-all.py --check 2>&1`
+FORMAT_RETVAL=$?
+
+if [ "$FORMAT_RETVAL" != "0" ]; then
+  echo -e "\n\n>>>> FAILED: format check; begin output\n\n"
+  echo -e "$FORMAT"
+  echo -e "\n\n>>>> FAILED: format check; end output\n\n"
+else
+  echo -e "\n\n>>>> PASSED: format check\n\n"
+fi
+
+RETVALS=(
+  $FORMAT_RETVAL
+)
+IFS=$'\n'
+RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1`
+
+exit $RETVAL

ci/cpu/build.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+##############################################
+# kvikIO CPU conda build script for CI       #
+##############################################
+set -e
+
+# Set path and build parallel level
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
+
+# Set home to the job's workspace
+export HOME="$WORKSPACE"
+
+# Determine CUDA release version
+export CUDA_REL=${CUDA_VERSION%.*}
+
+# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts)
+export GPUCI_CONDA_RETRY_MAX=1
+export GPUCI_CONDA_RETRY_SLEEP=30
+
+# Use Ninja to build, setup Conda Build Dir
+# export CMAKE_GENERATOR="Ninja"
+export CONDA_BLD_DIR="$WORKSPACE/.conda-bld"
+
+# Switch to project root; also root of repo checkout
+cd "$WORKSPACE"
+
+# If nightly build, append current YYMMDD to version
+if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then
+  export VERSION_SUFFIX=`date +%y%m%d`
+fi
+
+################################################################################
+# SETUP - Check environment
+################################################################################
+
+gpuci_logger "Check environment variables"
+env
+
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
+
+# Remove rapidsai-nightly channel if we are building main branch
+if [ "$SOURCE_BRANCH" = "main" ]; then
+  conda config --system --remove channels rapidsai-nightly
+fi
+
+gpuci_logger "Check compiler versions"
+python --version
+$CC --version
+$CXX --version
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
+
+# FIX Added to deal with Anancoda SSL verification issues during conda builds
+conda config --set ssl_verify False
+
+# FIXME: Remove
+gpuci_mamba_retry install -c conda-forge boa
+
+################################################################################
+# BUILD - Conda package builds
+################################################################################
+
+if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+  CONDA_BUILD_ARGS=""
+else
+  CONDA_BUILD_ARGS="--dirty --no-remove-work-dir"
+fi
+
+gpuci_logger "Build conda pkg for libkvikio"
+gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/libkvikio --python=$PYTHON $CONDA_BUILD_ARGS
+
+gpuci_logger "Build conda pkg for kvikio"
+gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/kvikio --python=$PYTHON $CONDA_BUILD_ARGS
+
+################################################################################
+# UPLOAD - Conda packages
+################################################################################
+
+gpuci_logger "Upload conda pkgs"
+source ci/cpu/upload.sh

ci/cpu/upload.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+##############################################
+# kvikIO CPU conda upload script for CI      #
+##############################################
+# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
+
+set -e
+
+# Setup 'gpuci_retry' for upload retries (results in 4 total attempts)
+export GPUCI_RETRY_MAX=3
+export GPUCI_RETRY_SLEEP=30
+
+# Set default label options if they are not defined elsewhere
+export LABEL_OPTION=${LABEL_OPTION:-"--label main"}
+
+# Skip uploads unless BUILD_MODE == "branch"
+if [ "${BUILD_MODE}" != "branch" ]; then
+  echo "Skipping upload"
+  return 0
+fi
+
+# Skip uploads if there is no upload key
+if [ -z "$MY_UPLOAD_KEY" ]; then
+  echo "No upload key"
+  return 0
+fi
+
+################################################################################
+# SETUP - Get conda file output locations
+################################################################################
+
+gpuci_logger "Get conda file output locations"
+export LIBKVIKIO_FILE=`conda build --no-build-id --croot "$WORKSPACE/.conda-bld" conda/recipes/libkvikio --output`
+export KVIKIO_FILE=$(conda build --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/kvikio --python=$PYTHON --output)
+
+################################################################################
+# UPLOAD - Conda packages
+################################################################################
+
+gpuci_logger "Starting conda uploads"
+gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBKVIKIO_FILE} --no-progress
+gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${KVIKIO_FILE} --no-progress

ci/docs/build.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+###################################
+# kvikIO Docs build script for CI #
+###################################
+
+if [ -z "$PROJECT_WORKSPACE" ]; then
+    echo ">>>> ERROR: Could not detect PROJECT_WORKSPACE in environment"
+    echo ">>>> WARNING: This script contains git commands meant for automated building, do not run locally"
+    exit 1
+fi
+
+export DOCS_WORKSPACE=$WORKSPACE/docs
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+export HOME=$WORKSPACE
+export PROJECT_WORKSPACE=/rapids/kvikio
+export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache"
+export PROJECTS=(kvikio libkvikio)
+
+gpuci_logger "Check environment"
+env
+
+gpuci_logger "Check GPU usage"
+nvidia-smi
+
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
+
+gpuci_logger "Install kvikIO"
+gpuci_conda_retry install -y -c rapidsai-nightly kvikio
+
+gpuci_logger "Check versions"
+python --version
+$CC --version
+$CXX --version
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
+
+# Build Doxygen docs
+gpuci_logger "Build Doxygen docs"
+cd $PROJECT_WORKSPACE/cpp/doxygen
+doxygen Doxyfile
+
+# Build Python docs
+gpuci_logger "Build Python docs"
+cd $PROJECT_WORKSPACE/docs
+make html
+
+#Commit to Website
+cd $DOCS_WORKSPACE
+
+for PROJECT in ${PROJECTS[@]}; do
+    if [ ! -d "api/$PROJECT/$BRANCH_VERSION" ]; then
+        mkdir -p api/$PROJECT/$BRANCH_VERSION
+    fi
+    rm -rf $DOCS_WORKSPACE/api/$PROJECT/$BRANCH_VERSION/*	
+done
+
+
+mv $PROJECT_WORKSPACE/cpp/doxygen/html/* $DOCS_WORKSPACE/api/libkvikio/$BRANCH_VERSION
+mv $PROJECT_WORKSPACE/docs/build/html/* $DOCS_WORKSPACE/api/kvikio/$BRANCH_VERSION
+

ci/gpu/build.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+##############################################
+# kvikIO GPU build and test script for CI    #
+##############################################
+set -e
+NUMARGS=$#
+ARGS=$*
+
+# Arg parsing function
+function hasArg {
+    (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
+}
+
+# Set path and build parallel level
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
+
+# Set home to the job's workspace
+export HOME="$WORKSPACE"
+
+# Switch to project root; also root of repo checkout
+cd "$WORKSPACE"
+
+# Determine CUDA release version
+export CUDA_REL=${CUDA_VERSION%.*}
+export CONDA_ARTIFACT_PATH="$WORKSPACE/ci/artifacts/kvikio/cpu/.conda-bld/"
+
+# Parse git describe
+export GIT_DESCRIBE_TAG=`git describe --tags`
+export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
+
+################################################################################
+# SETUP - Check environment
+################################################################################
+
+gpuci_logger "Check environment variables"
+env
+
+gpuci_logger "Check GPU usage"
+nvidia-smi
+
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
+
+gpuci_logger "Install dependencies"
+gpuci_mamba_retry install -y \
+                  "cudatoolkit=$CUDA_REL" \
+                  "cupy" \
+                  "zarr"
+
+gpuci_logger "Check compiler versions"
+python --version
+$CC --version
+$CXX --version
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
+
+################################################################################
+# TEST - Run py.test for kvikio
+################################################################################
+
+cd "$WORKSPACE/python"
+
+gpuci_logger "Build kvikio from source"
+python -m pip install .
+
+gpuci_logger "Python py.test for kvikio"
+py.test -n 6 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" --junitxml="$WORKSPACE/junit-kvikio.xml" -v
+
+if [ -n "${CODECOV_TOKEN}" ]; then
+    codecov -t $CODECOV_TOKEN
+fi
+
+return ${EXITCODE}