Skip to content

Commit

Permalink
ENH: Create unit tests for each python example pipeline (BlueQuartzSo…
Browse files Browse the repository at this point in the history
…ftware#793)

Signed-off-by: Michael Jackson <[email protected]>
  • Loading branch information
imikejackson committed Dec 26, 2023
1 parent 9aa1373 commit 3307b02
Show file tree
Hide file tree
Showing 25 changed files with 1,036 additions and 335 deletions.
13 changes: 11 additions & 2 deletions .github/workflows/linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,23 @@ jobs:
uses: actions/checkout@v3
with:
submodules: true
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
pip install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy
- name: Add C++ Problem Matcher
uses: ammaraskar/[email protected]
- name: Install Dependencies
- name: Install Dependencies - 2
run: |
sudo apt-get -y install ninja-build
- name: Install Sphinx
run: |
sudo pip install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy
sudo pip3 install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy
- name: Setup NuGet Credentials
shell: bash
run: |
Expand Down
13 changes: 11 additions & 2 deletions .github/workflows/macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,19 @@ jobs:
- name: Checkout
uses: actions/checkout@v3
with:
submodules: true
submodules: true
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
pip install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy
- name: Add C++ Problem Matcher
uses: ammaraskar/[email protected]
- name: Install Dependencies
- name: Install Dependencies - 2
run: |
brew install ninja
- name: Install Sphinx
Expand Down
28 changes: 28 additions & 0 deletions CMakePresets.json
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@
"type": "BOOL",
"value": "ON"
},
"SIMPLNX_ENABLE_PYTHON_TESTS": {
"type": "BOOL",
"value": "ON"
},
"SIMPLNX_EMBED_PYTHON": {
"type": "BOOL",
"value": "OFF"
Expand Down Expand Up @@ -121,6 +125,14 @@
"VCPKG_HOST_TRIPLET": {
"type": "STRING",
"value": "x64-osx-v11"
},
"Python3_EXECUTABLE": {
"type": "PATH",
"value": "/Users/runner/hostedtoolcache/Python/3.10.13/x64/bin/python3.10"
},
"SIMPLNX_PY_DISABLE_HIDDEN_VISIBILITY": {
"type": "BOOL",
"value": "ON"
}
}
},
Expand All @@ -138,6 +150,14 @@
"VCPKG_HOST_TRIPLET": {
"type": "STRING",
"value": "arm64-osx-dynamic"
},
"Python3_EXECUTABLE": {
"type": "PATH",
"value": "/Users/runner/hostedtoolcache/Python/3.10.13/x64/bin/python3.10"
},
"SIMPLNX_PY_DISABLE_HIDDEN_VISIBILITY": {
"type": "BOOL",
"value": "ON"
}
}
},
Expand All @@ -155,6 +175,10 @@
"VCPKG_HOST_TRIPLET": {
"type": "STRING",
"value": "x64-linux-dynamic"
},
"Python3_EXECUTABLE": {
"type": "PATH",
"value": "/opt/hostedtoolcache/Python/3.10.13/x64/bin/python3.10"
}
}
},
Expand Down Expand Up @@ -278,6 +302,10 @@
"SIMPLNX_PY_DISABLE_HIDDEN_VISIBILITY": {
"type": "BOOL",
"value": "ON"
},
"Python3_EXECUTABLE": {
"type": "PATH",
"value": "/opt/local/anaconda3/envs/nx-build/bin/python"
}
}
},
Expand Down
83 changes: 83 additions & 0 deletions cmake/Utility.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -460,3 +460,86 @@ function(cmpBuildDateRevisionString)
endif()

endfunction()


#-------------------------------------------------------------------------------
# @Brief function AddPythonTest
# @ NAME
# @ FILE
# @ PYTHONPATH
#-------------------------------------------------------------------------------
function(AddPythonTest)
set(options )
set(oneValueArgs NAME FILE)
set(multiValueArgs PYTHONPATH)
cmake_parse_arguments(ARGS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
message(STATUS "ARGS_FILE:${ARGS_FILE}")
if(COMPLEX_BUILD_PYTHON)
if(WIN32)
add_test(NAME ${ARGS_NAME}
COMMAND ${complex_SOURCE_DIR}/wrapping/python/testing/anaconda_test.bat
)

set_property(TEST ${ARGS_NAME}
PROPERTY
ENVIRONMENT
"PYTHON_TEST_FILE=${ARGS_FILE}"
"Python3_EXECUTABLE=${Python3_EXECUTABLE}"
)
else()
add_test(NAME ${ARGS_NAME}
COMMAND ${complex_SOURCE_DIR}/wrapping/python/testing/anaconda_test.sh
)
set_property(TEST ${ARGS_NAME}
PROPERTY
ENVIRONMENT
"PYTHON_TEST_FILE=${ARGS_FILE}"
"Python3_EXECUTABLE=${Python3_EXECUTABLE}"
)
endif()
else()
add_test(NAME ${ARGS_NAME}
COMMAND ${Python3_EXECUTABLE} ${ARGS_FILE}
)
endif()

if(WIN32)
string(REPLACE ";" "\\;" ARGS_PYTHONPATH "${ARGS_PYTHONPATH}")
else()
string(REPLACE ";" ":" ARGS_PYTHONPATH "${ARGS_PYTHONPATH}")
endif()

set_property(TEST ${ARGS_NAME}
APPEND
PROPERTY
ENVIRONMENT
"PYTHONPATH=${ARGS_PYTHONPATH}"
)
endfunction()

#-------------------------------------------------------------------------------
# @Brief function CreatePythonTests
# @ PREFIX
# @ INPUT_DIR
# @ TEST_NAMES
#-------------------------------------------------------------------------------
function(CreatePythonTests)
set(options)
set(oneValueArgs PREFIX INPUT_DIR)
set(multiValueArgs TEST_NAMES)
cmake_parse_arguments(ARGS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})

set(TESTS_PYTHONPATH
"$<TARGET_FILE_DIR:complex>"
)

foreach(test ${ARGS_TEST_NAMES})
string(REPLACE "/" "_" test_name ${test})
set(PY_TEST_NAME ${ARGS_PREFIX}_${test_name})

AddPythonTest(NAME ${PY_TEST_NAME}
FILE ${ARGS_INPUT_DIR}/${test}.py
PYTHONPATH ${TESTS_PYTHONPATH}
)
endforeach()
endfunction()
7 changes: 5 additions & 2 deletions src/Plugins/SimplnxCore/wrapping/python/simplnxpy.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -544,8 +544,11 @@ PYBIND11_MODULE(simplnx, mod)
std::vector<DataPath> outputPaths;
for(const auto* object : self.getTopLevelData())
{
auto topLevelPath = DataPath::FromString(object->getDataPaths()[0].getTargetName()).value();
outputPaths.push_back(topLevelPath);
if(object != nullptr)
{
auto topLevelPath = DataPath::FromString(object->getDataPaths()[0].getTargetName()).value();
outputPaths.push_back(topLevelPath);
}
}
return outputPaths;
}
Expand Down
6 changes: 3 additions & 3 deletions src/simplnx/Utilities/DataGroupUtilities.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,6 @@ std::optional<std::vector<DataPath>> GetAllChildDataPaths(const DataStructure& d
{
bool ignore = false;
DataPath childPath = parentGroup.createChildPath(childName);
const DataObject* dataObject = dataStructure.getData(childPath);
for(const auto& ignoredPath : ignoredDataPaths)
{
if(childPath == ignoredPath)
Expand All @@ -195,7 +194,8 @@ std::optional<std::vector<DataPath>> GetAllChildDataPaths(const DataStructure& d
break;
}
}
if(!ignore && (dataObjectType == DataObject::Type::DataObject || dataObject->getDataObjectType() == dataObjectType))
const DataObject* dataObject = dataStructure.getData(childPath);
if(dataObject != nullptr && !ignore && (dataObjectType == DataObject::Type::DataObject || dataObject->getDataObjectType() == dataObjectType))
{
childDataObjects.push_back(childPath);
}
Expand All @@ -211,7 +211,7 @@ std::optional<std::vector<DataPath>> GetAllChildDataPaths(const DataStructure& d
{
std::vector<DataPath> childDataObjects;
const DataObject* dataObject1 = dataStructure.getData(parent);
if(dataObject1->getDataObjectType() == DataObject::Type::DataArray || dataObject1->getDataObjectType() == DataObject::Type::DynamicListArray ||
if(dataObject1 == nullptr || dataObject1->getDataObjectType() == DataObject::Type::DataArray || dataObject1->getDataObjectType() == DataObject::Type::DynamicListArray ||
dataObject1->getDataObjectType() == DataObject::Type::NeighborList || dataObject1->getDataObjectType() == DataObject::Type::ScalarData ||
dataObject1->getDataObjectType() == DataObject::Type::StringArray)
{
Expand Down
47 changes: 45 additions & 2 deletions wrapping/python/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@ function(simplnx_create_pyi)
set(PYI_TARGET_NAME ${ARGS_PYTHON_MODULE_NAME}CreateStubFile)

add_custom_target(${PYI_TARGET_NAME} ALL
COMMAND ${MYPY_STUBGEN_EXE} -m ${ARGS_PYTHON_MODULE_NAME} -o $<TARGET_FILE_DIR:simplnx>
COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/fix_pyi_file.py --input_file "$<TARGET_FILE_DIR:simplnx>/${ARGS_PYTHON_MODULE_NAME}.pyi"
COMMAND "${MYPY_STUBGEN_EXE}" -m "${ARGS_PYTHON_MODULE_NAME}" -o "$<TARGET_FILE_DIR:simplnx>"
COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/fix_pyi_file.py" --input_file "$<TARGET_FILE_DIR:simplnx>/${ARGS_PYTHON_MODULE_NAME}.pyi"
COMMENT "${ARGS_PYTHON_MODULE_NAME}: Generating .pyi files"
WORKING_DIRECTORY $<TARGET_FILE_DIR:simplnx>
)
Expand Down Expand Up @@ -278,3 +278,46 @@ option(SIMPLNX_ENABLE_SPHINX_DOCS "Enables the Sphinx based documentation genera
if(SIMPLNX_ENABLE_SPHINX_DOCS)
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/docs ${simplnx_BINARY_DIR}/sphinx_docs)
endif()

#------------------------------------------------------------------------------
# Create the Python Unit tests
#------------------------------------------------------------------------------
option(COMPLEX_ENABLE_PYTHON_TESTS "Enables python based unit tests" OFF)
if(COMPLEX_ENABLE_PYTHON_TESTS)

file(READ ${complex_SOURCE_DIR}/wrapping/python/complex_test_dirs.in.py COMPLEX_TEST_DIRS_FILE)

string(REPLACE "\${CMAKE_LIBRARY_OUTPUT_DIRECTORY}" $<TARGET_FILE_DIR:complex> COMPLEX_TEST_DIRS_FILE ${COMPLEX_TEST_DIRS_FILE})
string(REPLACE "\${complex_BINARY_DIR}" ${complex_BINARY_DIR} COMPLEX_TEST_DIRS_FILE ${COMPLEX_TEST_DIRS_FILE})
string(REPLACE "\${DREAM3D_DATA_DIR}" ${DREAM3D_DATA_DIR} COMPLEX_TEST_DIRS_FILE ${COMPLEX_TEST_DIRS_FILE})
string(REPLACE "\${complex_SOURCE_DIR}" ${complex_SOURCE_DIR} COMPLEX_TEST_DIRS_FILE ${COMPLEX_TEST_DIRS_FILE})

file(GENERATE
OUTPUT "$<TARGET_FILE_DIR:complex>/complex_test_dirs.py"
CONTENT ${COMPLEX_TEST_DIRS_FILE}
)

set(PYTHON_TEST_INPUT_DIR "${complex_SOURCE_DIR}/wrapping/python/examples")

set(COMPLEX_PYTHON_TESTS
"angle_conversion"
"basic_arrays"
"basic_ebsd_ipf"
"basic_numpy"
"create_ensemble_info"
"generated_file_list"
"geometry_examples"
"import_d3d" # Dependent on 'basic_ebsd_ipf' running first
"import_hdf5" # Dependent on 'basic_ebsd_ipf' running first
"output_file"
"pipeline"
"read_csv_file"
# "read_esprit_data"
)

CreatePythonTests(PREFIX "PY_COMPLEX"
INPUT_DIR ${PYTHON_TEST_INPUT_DIR}
TEST_NAMES ${COMPLEX_PYTHON_TESTS}
)

endif()
13 changes: 12 additions & 1 deletion wrapping/python/CxPybind/CxPybind/CxPybind.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -397,7 +397,18 @@ auto BindFilter(py::handle scope, const Internals& internals)

std::string executeSig = MakePythonSignature<FilterT>("execute", internals);
std::string executeDocString = fmt::format("{}\n\nExecutes the filter\n", executeSig);

filter.def_static("human_name", [&internals]() {
FilterT filter;
return filter.humanName();
});
filter.def_static("name", [&internals]() {
FilterT filter;
return filter.name();
});
filter.def_static("uuid", [&internals]() {
FilterT filter;
return filter.uuid();
});
filter.def_static(
"execute",
[&internals](DataStructure& dataStructure, const py::kwargs& kwargs) {
Expand Down
50 changes: 50 additions & 0 deletions wrapping/python/complex_test_dirs.in.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
''' module to get build specific directories
This file is generated during the cmake configure phase. This file does *NOT*
however use the standard `cmake_configure_file()` command but instead uses
a custom generation cmake code. If you add new `${}` style variables, you will
need to update those custom CMake codes.
'''


import complex as cx

def check_filter_result(filter: cx.IFilter, result: cx.IFilter.ExecuteResult):
if len(result.warnings) != 0:
print(f'{filter.name()} :: Warnings: {result.warnings}')

has_errors = len(result.errors) != 0
if has_errors:
print(f'{filter.name()} :: Errors: {result.errors}')
raise RuntimeError(result)

print(f"{filter.name()} :: No errors running the filter")


def GetBuildDirectory():
return '${CMAKE_LIBRARY_OUTPUT_DIRECTORY}'

def GetTestDirectory():
return '${complex_BINARY_DIR}/Testing'

def GetTestTempDirectory():
return '${complex_BINARY_DIR}/Testing/Temporary'

def GetDataDirectory():
return '${DREAM3D_DATA_DIR}'

def GetComplexPythonSourceDir():
return '${complex_SOURCE_DIR}/wrapping/python'

def GetComplexSourceDir():
return '${complex_SOURCE_DIR}'

def print_all_paths():
print(f'#### Important Filesystem Paths ####')
print(f' GetBuildDirectory: {GetBuildDirectory()}')
print(f' GetTestDirectory: {GetTestDirectory()}')
print(f' GetTestTempDirectory: {GetTestTempDirectory()}')
print(f' GetDataDirectory: {GetDataDirectory()}')
print(f' GetComplexPythonSourceDir: {GetComplexPythonSourceDir()}')
print(f' GetComplexSourceDir: {GetComplexSourceDir()}')
print('#######################################')
Loading

0 comments on commit 3307b02

Please sign in to comment.