Skip to content

Commit

Permalink
PY: Fix GIL crashes, unit test false positive results, misc other tes…
Browse files Browse the repository at this point in the history
…t issues. (BlueQuartzSoftware#831)

- Switched to PYTHON_EXECUTABLE from Python3_EXECUTABLE in script
- Pybind11 uses PYTHON_EXECUTABLE so it should be defined
- Fixed test bat script to exit when python isn't found on windows
- Changed SIMPLNX_BUILD_PYTHON_TESTS to default to ON
- Fixed GIL state error in preflight/execute
- Changed simplnx_test_dirs functions to return Path
- Fixed directory used in basic_ebsd_ipf.py to use get_data_directory to give the appropriate path when using multiconfiguration generators

Signed-off-by: Jared Duffey <[email protected]>
  • Loading branch information
JDuffeyBQ authored Jan 30, 2024
1 parent 6406635 commit c611283
Show file tree
Hide file tree
Showing 62 changed files with 206 additions and 210 deletions.
32 changes: 13 additions & 19 deletions cmake/Utility.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -476,27 +476,21 @@ function(AddPythonTest)
# message(STATUS "ARGS_FILE:${ARGS_FILE}")
if(SIMPLNX_BUILD_PYTHON)
if(WIN32)
add_test(NAME ${ARGS_NAME}
COMMAND ${simplnx_SOURCE_DIR}/wrapping/python/testing/anaconda_test.bat
)

set_property(TEST ${ARGS_NAME}
PROPERTY
ENVIRONMENT
"PYTHON_TEST_FILE=${ARGS_FILE}"
"Python3_EXECUTABLE=${Python3_EXECUTABLE}"
)
set(test_driver_file ${simplnx_SOURCE_DIR}/wrapping/python/testing/anaconda_test.bat)
else()
add_test(NAME ${ARGS_NAME}
COMMAND ${simplnx_SOURCE_DIR}/wrapping/python/testing/anaconda_test.sh
)
set_property(TEST ${ARGS_NAME}
PROPERTY
ENVIRONMENT
"PYTHON_TEST_FILE=${ARGS_FILE}"
"Python3_EXECUTABLE=${Python3_EXECUTABLE}"
)
set(test_driver_file ${simplnx_SOURCE_DIR}/wrapping/python/testing/anaconda_test.sh)
endif()

add_test(NAME ${ARGS_NAME}
COMMAND ${test_driver_file}
)

set_property(TEST ${ARGS_NAME}
PROPERTY
ENVIRONMENT
"PYTHON_TEST_FILE=${ARGS_FILE}"
"PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}"
)
else()
add_test(NAME ${ARGS_NAME}
COMMAND ${Python3_EXECUTABLE} ${ARGS_FILE}
Expand Down
2 changes: 2 additions & 0 deletions src/Plugins/SimplnxCore/wrapping/python/simplnxpy.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1110,13 +1110,15 @@ PYBIND11_MODULE(simplnx, mod)
filter.def("human_name", &IFilter::humanName);
filter.def("preflight2", [internals](const IFilter& self, DataStructure& dataStructure, const py::kwargs& kwargs) {
Arguments convertedArgs = ConvertDictToArgs(*internals, self.parameters(), kwargs);
py::gil_scoped_release releaseGIL{};
IFilter::PreflightResult result = self.preflight(dataStructure, convertedArgs, CreatePyMessageHandler());
return result;
});
filter.def(
"execute2",
[internals](const IFilter& self, DataStructure& dataStructure, const py::kwargs& kwargs) {
Arguments convertedArgs = ConvertDictToArgs(*internals, self.parameters(), kwargs);
py::gil_scoped_release releaseGIL{};
IFilter::ExecuteResult result = self.execute(dataStructure, convertedArgs, nullptr, CreatePyMessageHandler());
return result;
},
Expand Down
3 changes: 1 addition & 2 deletions wrapping/python/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -282,9 +282,8 @@ endif()
#------------------------------------------------------------------------------
# Create the Python Unit tests
#------------------------------------------------------------------------------
option(SIMPLNX_BUILD_PYTHON_TESTS "Enables python based unit tests" OFF)
option(SIMPLNX_BUILD_PYTHON_TESTS "Enables python based unit tests" ON)
if(SIMPLNX_BUILD_PYTHON_TESTS)

get_property(is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
if(is_multi_config)
set(CX_CONFIG_DIR "$<CONFIG>")
Expand Down
2 changes: 2 additions & 0 deletions wrapping/python/CxPybind/CxPybind/CxPybind.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,7 @@ std::string MakePythonSignature(std::string_view funcName, const Internals& inte

inline void PyPrintMessage(const IFilter::Message& message)
{
py::gil_scoped_acquire acquireGIL{};
py::print(fmt::format("{}", message.message));
}

Expand Down Expand Up @@ -426,6 +427,7 @@ auto BindFilter(py::handle scope, const Internals& internals)
}

Arguments convertedArgs = ConvertDictToArgs(internals, filter.parameters(), kwargs);
py::gil_scoped_release releaseGIL{};
IFilter::ExecuteResult result = filter.execute(dataStructure, convertedArgs, nullptr, CreatePyMessageHandler());
return result;
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# Filter 1

generated_file_list_value = nx.GeneratedFileListParameter.ValueType()
generated_file_list_value.input_path = nxtest.GetDataDirectory() + "/Porosity_Image/"
generated_file_list_value.input_path = str(nxtest.get_data_directory() / "Porosity_Image/")
generated_file_list_value.ordering = nx.GeneratedFileListParameter.Ordering.LowToHigh

generated_file_list_value.file_prefix = "slice_"
Expand Down Expand Up @@ -183,7 +183,7 @@
# Instantiate Filter
nx_filter = nx.WriteDREAM3DFilter()
# Execute Filter with Parameters
output_file_path = nxtest.GetDataDirectory() + "/Output/ImagesStack/Images.dream3d"
output_file_path = nxtest.get_data_directory() / "Output/ImagesStack/Images.dream3d"
result = nx_filter.execute(data_structure=data_structure,
export_file_path=output_file_path,
write_xdmf_file=True)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
nx_filter = cxitk.ITKImportImageStack()

generated_file_list_value = nx.GeneratedFileListParameter.ValueType()
generated_file_list_value.input_path = nxtest.GetDataDirectory() + "/Porosity_Image/"
generated_file_list_value.input_path = str(nxtest.get_data_directory() / "Porosity_Image/")
generated_file_list_value.ordering = nx.GeneratedFileListParameter.Ordering.LowToHigh

generated_file_list_value.file_prefix = "slice_"
Expand Down Expand Up @@ -243,7 +243,7 @@
# Instantiate Filter
nx_filter = nx.WriteDREAM3DFilter()
# Execute Filter with Parameters
output_file_path = nxtest.GetDataDirectory() + "/Output/Porosity_Analysis.dream3d"
output_file_path = nxtest.get_data_directory() / "Output/Porosity_Analysis.dream3d"
result = nx_filter.execute(
data_structure=data_structure,
export_file_path=output_file_path,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

# Filter 2
# Define output file path
output_file_path = nxtest.GetDataDirectory() + "/Output/H5EspritData/H5EspritData.dream3d"
output_file_path = nxtest.get_data_directory() / "Output/H5EspritData/H5EspritData.dream3d"
# Instantiate Filter
nx_filter = cx.WriteDREAM3DFilter()
# Execute WriteDREAM3DFilter with Parameters
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@

# Filter 4
# Define output file path
output_file_path = nxtest.GetDataDirectory() + "/Output/EdaxOIMData/EdaxOIMData.dream3d"
output_file_path = nxtest.get_data_directory() / "Output/EdaxOIMData/EdaxOIMData.dream3d"
# Instantiate Filter
nx_filter = cx.WriteDREAM3DFilter()
# Execute WriteDREAM3DFilter with Parameters
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# Instantiate Filter

generated_file_list_value = nx.GeneratedFileListParameter.ValueType()
generated_file_list_value.input_path = nxtest.GetDataDirectory() + "/Small_IN100"
generated_file_list_value.input_path = str(nxtest.get_data_directory() / "Small_IN100")
generated_file_list_value.ordering = nx.GeneratedFileListParameter.Ordering.HighToLow

generated_file_list_value.file_prefix = "Slice_"
Expand All @@ -29,7 +29,7 @@
result = nx_filter.execute(
data_structure=data_structure,
input_file_list_info=generated_file_list_value,
output_file_path=nxtest.GetDataDirectory() + "/Output/Reconstruction/Small_IN100.h5ebsd",
output_file_path=nxtest.get_data_directory() / "Output/Reconstruction/Small_IN100.h5ebsd",
reference_frame=0,
stacking_order=1,
z_spacing=0.25
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# Filter 1
# Instantiate Import Data Parameter
import_data = nx.Dream3dImportParameter.ImportData()
import_data.file_path = nxtest.GetDataDirectory() + "/Output/Reconstruction/SmallIN100_Final.dream3d"
import_data.file_path = str(nxtest.get_data_directory() / "Output/Reconstruction/SmallIN100_Final.dream3d")
import_data.data_paths = None

# Instantiate Filter
Expand Down Expand Up @@ -165,7 +165,7 @@

# Filter 10
# Set Output File Path
output_file_path = nxtest.GetDataDirectory() + "/Output/Statistics/SmallIN100_Morph.dream3d"
output_file_path = nxtest.get_data_directory() / "Output/Statistics/SmallIN100_Morph.dream3d"
# Instantiate Filter
nx_filter = nx.WriteDREAM3DFilter()
# Execute Filter with Parameters
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# Filter 1
# Instantiate Import Data Parameter
import_data = nx.Dream3dImportParameter.ImportData()
import_data.file_path = nxtest.GetDataDirectory() + "/Output/Statistics/SmallIN100_CrystalStats.dream3d"
import_data.file_path = str(nxtest.get_data_directory() / "Output/Statistics/SmallIN100_CrystalStats.dream3d")
import_data.data_paths = None
# Instantiate Filter
nx_filter = nx.ReadDREAM3DFilter()
Expand Down Expand Up @@ -70,7 +70,7 @@

# Filter 5
# Set Output File Path
output_file_path = nxtest.GetDataDirectory() + "/Output/SurfaceMesh/SmallIN100_Mesh.dream3d"
output_file_path = nxtest.get_data_directory() / "Output/SurfaceMesh/SmallIN100_Mesh.dream3d"
# Instantiate Filter
nx_filter = nx.WriteDREAM3DFilter()
# Execute Filter with Parameters
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# Filter 1
# Instantiate Import Data Parameter
import_data = nx.Dream3dImportParameter.ImportData()
import_data.file_path = nxtest.GetDataDirectory() + "/Output/SurfaceMesh/SmallIN100_Mesh.dream3d"
import_data.file_path = str(nxtest.get_data_directory() / "Output/SurfaceMesh/SmallIN100_Mesh.dream3d")
import_data.data_paths = None
# Instantiate Filter
nx_filter = nx.ReadDREAM3DFilter()
Expand Down Expand Up @@ -44,7 +44,7 @@
# Instantiate Filter
nx_filter = nx.WriteDREAM3DFilter()
# Set Output File Path
output_file_path = nxtest.GetDataDirectory() + "/Output/SurfaceMesh/SmallIN100_Smoothed.dream3d"
output_file_path = nxtest.get_data_directory() / "Output/SurfaceMesh/SmallIN100_Smoothed.dream3d"
# Execute Filter with Parameters
result = nx_filter.execute(data_structure=data_structure,
export_file_path=output_file_path,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# Filter 1
# Instantiate Import Data Parameter
import_data = nx.Dream3dImportParameter.ImportData()
import_data.file_path = nxtest.GetDataDirectory() + "/Output/SurfaceMesh/SmallIN100_Smoothed.dream3d"
import_data.file_path = str(nxtest.get_data_directory() / "Output/SurfaceMesh/SmallIN100_Smoothed.dream3d")
import_data.data_paths = None
# Instantiate Filter
nx_filter = nx.ReadDREAM3DFilter()
Expand Down Expand Up @@ -102,7 +102,7 @@
# Instantiate Filter
nx_filter = nx.WriteDREAM3DFilter()
# Execute Filter with Parameters
output_file_path = nxtest.GetDataDirectory() + "/Output/SurfaceMesh/SmallIN100_MeshStats.dream3d"
output_file_path = nxtest.get_data_directory() / "Output/SurfaceMesh/SmallIN100_MeshStats.dream3d"
result = nx_filter.execute(
data_structure=data_structure,
export_file_path=output_file_path,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# Filter 1
# Instantiate Import Data Parameter
import_data = nx.Dream3dImportParameter.ImportData()
import_data.file_path = nxtest.GetDataDirectory() + "/Output/SurfaceMesh/SmallIN100_MeshStats.dream3d"
import_data.file_path = str(nxtest.get_data_directory() / "Output/SurfaceMesh/SmallIN100_MeshStats.dream3d")
import_data.data_paths = None
# Instantiate Filter
nx_filter = nx.ReadDREAM3DFilter()
Expand Down Expand Up @@ -123,7 +123,7 @@
# Instantiate Filter
nx_filter = nx.WriteDREAM3DFilter()
# Set Output File Path
output_file_path = nxtest.GetDataDirectory() + "/Output/SurfaceMesh/SmallIN100_GBCD.dream3d"
output_file_path = nxtest.get_data_directory() / "Output/SurfaceMesh/SmallIN100_GBCD.dream3d"
# Execute Filter with Parameters
result = nx_filter.execute(data_structure=data_structure,
export_file_path=output_file_path,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# Filter 1
# Instantiate Import Data Parameter
import_data = nx.Dream3dImportParameter.ImportData()
import_data.file_path = nxtest.GetDataDirectory() + "/Output/Statistics/SmallIN100_Morph.dream3d"
import_data.file_path = str(nxtest.get_data_directory() / "Output/Statistics/SmallIN100_Morph.dream3d")
import_data.data_paths = None
# Instantiate Filter
nx_filter = nx.ReadDREAM3DFilter()
Expand Down Expand Up @@ -135,7 +135,7 @@
# Instantiate Filter
nx_filter = nx.WriteDREAM3DFilter()
# Set Output File Path
output_file_path = nxtest.GetDataDirectory() + "/Output/Statistics/SmallIN100_CrystalStats.dream3d"
output_file_path = nxtest.get_data_directory() / "Output/Statistics/SmallIN100_CrystalStats.dream3d"
# Execute Filter with Parameters
result = nx_filter.execute(
data_structure=data_structure,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# Filter 1
# Instantiate Import Data Parameter
import_data = nx.Dream3dImportParameter.ImportData()
import_data.file_path = nxtest.GetDataDirectory() + "/Output/SurfaceMesh/SmallIN100_MeshStats.dream3d"
import_data.file_path = str(nxtest.get_data_directory() / "Output/SurfaceMesh/SmallIN100_MeshStats.dream3d")
import_data.data_paths =[
nx.DataPath("DataContainer"),
nx.DataPath("DataContainer/CellData"),
Expand Down Expand Up @@ -126,7 +126,7 @@
# Instantiate Filter
nx_filter = nx.WriteDREAM3DFilter()
# Set Output File Path
output_file_path = nxtest.GetDataDirectory() + "/Output/SurfaceMesh/SmallIN100_GBCD_Metric.dream3d"
output_file_path = nxtest.get_data_directory() / "Output/SurfaceMesh/SmallIN100_GBCD_Metric.dream3d"
# Execute Filter with Parameters
result = nx_filter.execute(
data_structure=data_structure,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
filter_parameter.euler_representation=0
filter_parameter.end_slice=117
filter_parameter.selected_array_names=["Confidence Index", "EulerAngles", "Fit", "Image Quality", "Phases", "SEM Signal", "X Position", "Y Position"]
filter_parameter.input_file_path=nxtest.GetDataDirectory() + "/Output/Reconstruction/Small_IN100.h5ebsd"
filter_parameter.input_file_path=str(nxtest.get_data_directory() / "Output/Reconstruction/Small_IN100.h5ebsd")
filter_parameter.start_slice=6
filter_parameter.use_recommended_transform=True

Expand Down Expand Up @@ -386,7 +386,7 @@
# Instantiate Filter
nx_filter = nx.WriteDREAM3DFilter()
# Set Output File Path
output_file_path = nxtest.GetDataDirectory() + "/Output/Reconstruction/SmallIN100_Final.dream3d"
output_file_path = nxtest.get_data_directory() / "Output/Reconstruction/SmallIN100_Final.dream3d"
# Execute Filter with Parameters
result = nx_filter.execute(
data_structure=data_structure,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
data_container_name=nx.DataPath("fw-ar-IF1-aptr12-corr"),
degrees_to_radians=True,
edax_hexagonal_alignment=True,
input_file=nxtest.GetDataDirectory() + "/T12-MAI-2010/fw-ar-IF1-aptr12-corr.ctf"
input_file=nxtest.get_data_directory() / "T12-MAI-2010/fw-ar-IF1-aptr12-corr.ctf"
)
nxtest.check_filter_result(nx_filter, result)

Expand Down Expand Up @@ -110,7 +110,7 @@
# Execute Filter with Parameters
result = nx_filter.execute(
data_structure=data_structure,
file_name=nxtest.GetDataDirectory() + "/Output/fw-ar-IF1-aptr12-corr/fw-ar-IF1-aptr12-corr_001.png",
file_name=nxtest.get_data_directory() / "Output/fw-ar-IF1-aptr12-corr/fw-ar-IF1-aptr12-corr_001.png",
image_array_path=nx.DataPath("fw-ar-IF1-aptr12-corr/Cell Data/IPF_001"),
image_geom_path=nx.DataPath("fw-ar-IF1-aptr12-corr"),
index_offset=0,
Expand Down Expand Up @@ -140,7 +140,7 @@
# Execute Filter with Parameters
result = nx_filter.execute(
data_structure=data_structure,
file_name=nxtest.GetDataDirectory() + "/Output/fw-ar-IF1-aptr12-corr/fw-ar-IF1-aptr12-corr_010.png",
file_name=nxtest.get_data_directory() / "Output/fw-ar-IF1-aptr12-corr/fw-ar-IF1-aptr12-corr_010.png",
image_array_path=nx.DataPath("fw-ar-IF1-aptr12-corr/Cell Data/IPF_010"),
image_geom_path=nx.DataPath("fw-ar-IF1-aptr12-corr"),
index_offset=0,
Expand Down Expand Up @@ -169,7 +169,7 @@
# Execute Filter with Parameters
result = nx_filter.execute(
data_structure=data_structure,
file_name=nxtest.GetDataDirectory() + "/Output/fw-ar-IF1-aptr12-corr/fw-ar-IF1-aptr12-corr_100.png",
file_name=nxtest.get_data_directory() / "Output/fw-ar-IF1-aptr12-corr/fw-ar-IF1-aptr12-corr_100.png",
image_array_path=nx.DataPath("fw-ar-IF1-aptr12-corr/Cell Data/IPF_100"),
image_geom_path=nx.DataPath("fw-ar-IF1-aptr12-corr"),
index_offset=0,
Expand Down Expand Up @@ -331,7 +331,7 @@
data_structure=data_structure,
cell_feature_attribute_matrix_path=nx.DataPath("fw-ar-IF1-aptr12-corr/CellFeatureData"),
delimiter_choice_int=2,
feature_data_file=nxtest.GetDataDirectory() + "/Output/fw-ar-IF1-aptr12-corr/FeatureData.csv",
feature_data_file=nxtest.get_data_directory() / "Output/fw-ar-IF1-aptr12-corr/FeatureData.csv",
write_neighborlist_data=False,
write_num_features_line=True
)
Expand Down Expand Up @@ -359,7 +359,7 @@
result = nx_filter.execute(
data_structure=data_structure,
delimiter=2,
output_path=nxtest.GetDataDirectory() + "/Output/fw-ar-IF1-aptr12-corr/EqDiamHistogram.csv",
output_path=nxtest.get_data_directory() / "Output/fw-ar-IF1-aptr12-corr/EqDiamHistogram.csv",
output_style=1,
selected_data_array_paths=[nx.DataPath("fw-ar-IF1-aptr12-corr/Histograms/EquivalentDiameters Histogram")]
)
Expand All @@ -368,7 +368,7 @@
# Filter 24
# Instantiate Filter
nx_filter = nx.WriteDREAM3DFilter()
output_file_path = nxtest.GetDataDirectory() + "/Output/fw-ar-IF1-aptr12-corr/fw-ar-IF1-aptr12-corr.dream3d"
output_file_path = nxtest.get_data_directory() / "Output/fw-ar-IF1-aptr12-corr/fw-ar-IF1-aptr12-corr.dream3d"

# Execute Filter with Parameters
result = nx_filter.execute(
Expand All @@ -382,7 +382,7 @@
# THIS SECTION IS ONLY HERE FOR CLEANING UP THE CI Machines
# If you are using this code, you should COMMENT out the next line
nxtest.cleanup_test_file(output_file_path)
nxtest.cleanup_test_dir(nxtest.GetDataDirectory() + "/Output/fw-ar-IF1-aptr12-corr/")
nxtest.cleanup_test_dir(nxtest.get_data_directory() / "Output/fw-ar-IF1-aptr12-corr/")
# *****************************************************************************

print("===> Pipeline Complete")
Loading

0 comments on commit c611283

Please sign in to comment.