From d0b11e8bb47dacb6e200ac8b0e2609626ccee0f3 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 4 Mar 2025 11:35:29 -0800 Subject: [PATCH 1/9] Update [ghstack-poisoned] --- CMakeLists.txt | 2 +- build/cmake_deps.toml | 18 ++++++++++++++++++ kernels/optimized/CMakeLists.txt | 2 +- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index de941663a88..73b89b6171e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -749,9 +749,9 @@ endif() if(EXECUTORCH_BUILD_PTHREADPOOL AND EXECUTORCH_BUILD_CPUINFO - AND CMAKE_CXX_STANDARD GREATER_EQUAL 14 ) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/threadpool) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/parallel) endif() if(EXECUTORCH_BUILD_PYBIND) diff --git a/build/cmake_deps.toml b/build/cmake_deps.toml index 4b22a09cb5b..4bbfd636a96 100644 --- a/build/cmake_deps.toml +++ b/build/cmake_deps.toml @@ -73,6 +73,7 @@ excludes = [ deps = [ "executorch", "executorch_core", + "extension_parallel", "extension_threadpool", "portable_kernels", ] @@ -115,6 +116,7 @@ excludes = [ deps = [ "executorch_core", "executorch", + "extension_parallel", ] [targets.optimized_native_cpu_ops] @@ -129,6 +131,8 @@ excludes = [ deps = [ "executorch_core", "executorch", + "extension_parallel", + "extension_threadpool", "portable_kernels", ] # ---------------------------------- core end ---------------------------------- @@ -208,6 +212,19 @@ deps = [ "extension_runner_util", ] +[targets.extension_parallel] +buck_targets = [ + "//extension/parallel:thread_parallel", +] +filters = [ + ".cpp$", +] +deps = [ + "executorch", + "executorch_core", + "extension_threadpool", +] + [targets.extension_tensor] buck_targets = [ "//extension/tensor:tensor", @@ -395,6 +412,7 @@ deps = [ "executorch", "executorch_core", "optimized_kernels", + "extension_parallel", "extension_threadpool", "xnnpack_backend", ] diff --git a/kernels/optimized/CMakeLists.txt b/kernels/optimized/CMakeLists.txt index 235c6738d9a..d9b19d4f9c2 100644 --- a/kernels/optimized/CMakeLists.txt +++ b/kernels/optimized/CMakeLists.txt @@ -43,7 +43,7 @@ endif() list(TRANSFORM _optimized_cpublas__srcs PREPEND "${EXECUTORCH_ROOT}/") add_library(cpublas STATIC ${_optimized_cpublas__srcs}) target_link_libraries( - cpublas PRIVATE executorch_core eigen_blas extension_threadpool + cpublas PRIVATE executorch_core eigen_blas extension_parallel extension_threadpool ) target_compile_options(cpublas PUBLIC ${_common_compile_options}) From 9437be1e7055d5705540d3544955b5d30f72be43 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 4 Mar 2025 12:29:58 -0800 Subject: [PATCH 2/9] Update [ghstack-poisoned] --- extension/parallel/CMakeLists.txt | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 extension/parallel/CMakeLists.txt diff --git a/extension/parallel/CMakeLists.txt b/extension/parallel/CMakeLists.txt new file mode 100644 index 00000000000..7f727aafe46 --- /dev/null +++ b/extension/parallel/CMakeLists.txt @@ -0,0 +1,25 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Please keep this file formatted by running: +# ~~~ +# cmake-format -i CMakeLists.txt +# ~~~ + +if(NOT (EXECUTORCH_BUILD_PTHREADPOOL AND EXECUTORCH_BUILD_CPUINFO)) + message(FATAL_ERROR "extension/parallel requires extension/threadpool") +endif() + +add_library(extension_parallel thread_parallel.cpp) + +target_link_libraries(extension_parallel PUBLIC executorch_core extension_threadpool) +target_compile_options(extension_parallel PUBLIC ${_common_compile_options}) + +install( + TARGETS extension_parallel + DESTINATION lib + INCLUDES + DESTINATION ${_common_include_directories}) From 643e10ee081b1ea34f0f5fb49f7df44a9f2f666b Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 4 Mar 2025 14:54:47 -0800 Subject: [PATCH 3/9] Update [ghstack-poisoned] --- build/executorch-config.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build/executorch-config.cmake b/build/executorch-config.cmake index d14a1227cd9..539c7c2960e 100644 --- a/build/executorch-config.cmake +++ b/build/executorch-config.cmake @@ -67,6 +67,7 @@ set(lib_list portable_ops_lib extension_module extension_module_static + extension_parallel extension_runner_util extension_tensor extension_threadpool @@ -114,3 +115,7 @@ foreach(lib ${lib_list}) list(APPEND EXECUTORCH_LIBRARIES ${lib}) endif() endforeach() + +# TODO: investigate use of install(EXPORT) to cleanly handle +# target_compile_options/target_compile_definitions for everything. +target_link_libraries(cpublas INTERFACE extension_parallel) From 6f2842b876a5a2310c6bf7311d6f6b76bc54549e Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 4 Mar 2025 14:54:48 -0800 Subject: [PATCH 4/9] Update [ghstack-poisoned] --- build/executorch-config.cmake | 1 + examples/models/llama/CMakeLists.txt | 9 --------- examples/models/llava/CMakeLists.txt | 9 --------- 3 files changed, 1 insertion(+), 18 deletions(-) diff --git a/build/executorch-config.cmake b/build/executorch-config.cmake index 539c7c2960e..d238db8ca95 100644 --- a/build/executorch-config.cmake +++ b/build/executorch-config.cmake @@ -65,6 +65,7 @@ set(lib_list neuron_backend qnn_executorch_backend portable_ops_lib + custom_ops extension_module extension_module_static extension_parallel diff --git a/examples/models/llama/CMakeLists.txt b/examples/models/llama/CMakeLists.txt index 5f49581ea25..f5d5a78d430 100644 --- a/examples/models/llama/CMakeLists.txt +++ b/examples/models/llama/CMakeLists.txt @@ -84,14 +84,6 @@ if(CMAKE_TOOLCHAIN_IOS OR ANDROID) target_link_options_shared_lib(executorch) endif() -# custom ops library -if(EXECUTORCH_BUILD_KERNELS_CUSTOM) - add_subdirectory( - ${CMAKE_CURRENT_SOURCE_DIR}/../../../extension/llm/custom_ops - ${CMAKE_CURRENT_BINARY_DIR}/../../../extension/llm/custom_ops - ) -endif() - # llama_runner library add_subdirectory(runner) @@ -119,7 +111,6 @@ target_link_options_shared_lib(quantized_ops_lib) list(APPEND link_libraries quantized_kernels quantized_ops_lib) if(EXECUTORCH_BUILD_KERNELS_CUSTOM) - target_link_options_shared_lib(custom_ops) list(APPEND link_libraries custom_ops) endif() diff --git a/examples/models/llava/CMakeLists.txt b/examples/models/llava/CMakeLists.txt index ecd00809fdb..f7fa4bacc04 100644 --- a/examples/models/llava/CMakeLists.txt +++ b/examples/models/llava/CMakeLists.txt @@ -93,14 +93,6 @@ if(CMAKE_TOOLCHAIN_IOS OR ANDROID) target_link_options_shared_lib(executorch) endif() -# custom ops library -if(EXECUTORCH_BUILD_KERNELS_CUSTOM) - add_subdirectory( - ${EXECUTORCH_ROOT}/extension/llm/custom_ops - ${CMAKE_CURRENT_BINARY_DIR}/../../../extension/llm/custom_ops - ) -endif() - # llava_runner library add_subdirectory(runner) @@ -132,7 +124,6 @@ target_link_options_shared_lib(quantized_ops_lib) list(APPEND link_libraries quantized_kernels quantized_ops_lib) if(EXECUTORCH_BUILD_KERNELS_CUSTOM) - target_link_options_shared_lib(custom_ops) list(APPEND link_libraries custom_ops) endif() From e47dfeb68a2e8e2ae0e3fa2add553f724f73404d Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 4 Mar 2025 14:54:52 -0800 Subject: [PATCH 5/9] Update [ghstack-poisoned] --- build/executorch-config.cmake | 1 + examples/models/llama/CMakeLists.txt | 1 - examples/models/llava/CMakeLists.txt | 1 - examples/models/llava/targets.bzl | 3 --- extension/android/CMakeLists.txt | 1 - extension/llm/custom_ops/CMakeLists.txt | 4 ++-- extension/threadpool/CMakeLists.txt | 1 + 7 files changed, 4 insertions(+), 8 deletions(-) diff --git a/build/executorch-config.cmake b/build/executorch-config.cmake index d238db8ca95..75e1075d929 100644 --- a/build/executorch-config.cmake +++ b/build/executorch-config.cmake @@ -120,3 +120,4 @@ endforeach() # TODO: investigate use of install(EXPORT) to cleanly handle # target_compile_options/target_compile_definitions for everything. target_link_libraries(cpublas INTERFACE extension_parallel) +target_compile_definitions(extension_threadpool INTERFACE ET_USE_THREADPOOL) diff --git a/examples/models/llama/CMakeLists.txt b/examples/models/llama/CMakeLists.txt index f5d5a78d430..b3364be610a 100644 --- a/examples/models/llama/CMakeLists.txt +++ b/examples/models/llama/CMakeLists.txt @@ -131,7 +131,6 @@ endif() set(XNNPACK_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../backends/xnnpack) # Extra compile option and include dir for pthreadpool if(EXECUTORCH_BUILD_PTHREADPOOL) - list(APPEND _common_compile_options -DET_USE_THREADPOOL) list(APPEND link_libraries extension_threadpool pthreadpool) list(APPEND _common_include_directories ${XNNPACK_ROOT}/third-party/pthreadpool/include diff --git a/examples/models/llava/CMakeLists.txt b/examples/models/llava/CMakeLists.txt index f7fa4bacc04..5d5857dd5af 100644 --- a/examples/models/llava/CMakeLists.txt +++ b/examples/models/llava/CMakeLists.txt @@ -130,7 +130,6 @@ endif() set(XNNPACK_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../backends/xnnpack) # Extra compile option and include dir for pthreadpool if(EXECUTORCH_BUILD_PTHREADPOOL) - list(APPEND _common_compile_options -DET_USE_THREADPOOL) list(APPEND link_libraries extension_threadpool pthreadpool) list(APPEND _common_include_directories ${XNNPACK_ROOT}/third-party/pthreadpool/include diff --git a/examples/models/llava/targets.bzl b/examples/models/llava/targets.bzl index 5efb099f06f..6f3a370acf4 100644 --- a/examples/models/llava/targets.bzl +++ b/examples/models/llava/targets.bzl @@ -7,9 +7,6 @@ def define_common_targets(): "main.cpp", ], compiler_flags = ["-Wno-global-constructors"], - preprocessor_flags = [ - "-DET_USE_THREADPOOL", - ], deps = [ "//executorch/examples/models/llava/runner:runner", "//executorch/extension/evalue_util:print_evalue", diff --git a/extension/android/CMakeLists.txt b/extension/android/CMakeLists.txt index 70f21f2751c..849d1d14364 100644 --- a/extension/android/CMakeLists.txt +++ b/extension/android/CMakeLists.txt @@ -124,7 +124,6 @@ if(EXECUTORCH_BUILD_KERNELS_CUSTOM) endif() if(TARGET pthreadpool) - target_compile_definitions(executorch_jni PRIVATE ET_USE_THREADPOOL=1) target_include_directories( executorch_jni PUBLIC diff --git a/extension/llm/custom_ops/CMakeLists.txt b/extension/llm/custom_ops/CMakeLists.txt index c3969e6f9bf..eeb118d4344 100644 --- a/extension/llm/custom_ops/CMakeLists.txt +++ b/extension/llm/custom_ops/CMakeLists.txt @@ -78,7 +78,7 @@ target_include_directories( target_link_libraries(custom_ops PUBLIC ${custom_ops_libs} executorch_core) target_compile_options( - custom_ops PUBLIC ${_common_compile_options} -DET_USE_THREADPOOL + custom_ops PUBLIC ${_common_compile_options} ) install(TARGETS custom_ops DESTINATION lib) @@ -130,7 +130,7 @@ if(EXECUTORCH_BUILD_KERNELS_CUSTOM_AOT) target_compile_options( custom_ops_aot_lib PUBLIC -Wno-deprecated-declarations -fPIC -frtti -fexceptions - ${_common_compile_options} -DET_USE_THREADPOOL + ${_common_compile_options} ) install(TARGETS custom_ops_aot_lib diff --git a/extension/threadpool/CMakeLists.txt b/extension/threadpool/CMakeLists.txt index 90288656674..c1d86acf75d 100644 --- a/extension/threadpool/CMakeLists.txt +++ b/extension/threadpool/CMakeLists.txt @@ -32,6 +32,7 @@ target_include_directories( PUBLIC ${EXECUTORCH_ROOT}/backends/xnnpack/third-party/cpuinfo/include ${EXECUTORCH_ROOT}/backends/xnnpack/third-party/pthreadpool/include ) +target_compile_definitions(extension_threadpool PUBLIC ET_USE_THREADPOOL) target_compile_options(extension_threadpool PUBLIC ${_common_compile_options}) # Install libraries From d4c7667877decc910d1341869f50c58506139e80 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 4 Mar 2025 14:54:56 -0800 Subject: [PATCH 6/9] Update [ghstack-poisoned] --- examples/models/llama/main.cpp | 2 ++ examples/models/llava/main.cpp | 2 ++ extension/android/jni/jni_layer_llama.cpp | 2 ++ extension/llm/custom_ops/op_sdpa.cpp | 2 ++ 4 files changed, 8 insertions(+) diff --git a/examples/models/llama/main.cpp b/examples/models/llama/main.cpp index 5fe0ce93cf6..e91d71e4dfb 100644 --- a/examples/models/llama/main.cpp +++ b/examples/models/llama/main.cpp @@ -13,6 +13,8 @@ #if defined(ET_USE_THREADPOOL) #include #include +#else +#error "threadpool is required" #endif DEFINE_string( diff --git a/examples/models/llava/main.cpp b/examples/models/llava/main.cpp index b01b33f5dd8..22e40d64bc8 100644 --- a/examples/models/llava/main.cpp +++ b/examples/models/llava/main.cpp @@ -17,6 +17,8 @@ #if defined(ET_USE_THREADPOOL) #include #include +#else +#error "threadpool is required" #endif DEFINE_string( diff --git a/extension/android/jni/jni_layer_llama.cpp b/extension/android/jni/jni_layer_llama.cpp index 5c58c89ee91..861497f333b 100644 --- a/extension/android/jni/jni_layer_llama.cpp +++ b/extension/android/jni/jni_layer_llama.cpp @@ -24,6 +24,8 @@ #if defined(ET_USE_THREADPOOL) #include #include +#else +#error "threadpool is required" #endif #include diff --git a/extension/llm/custom_ops/op_sdpa.cpp b/extension/llm/custom_ops/op_sdpa.cpp index f0a7775e803..afaa85534fc 100644 --- a/extension/llm/custom_ops/op_sdpa.cpp +++ b/extension/llm/custom_ops/op_sdpa.cpp @@ -21,6 +21,8 @@ #ifdef ET_USE_THREADPOOL #include #include +#else +#error "threadpool is required" #endif #include From a92958a1f6fbfbc154ec24c6b4ee6c6ebd41aea8 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 4 Mar 2025 18:41:25 -0800 Subject: [PATCH 7/9] Update [ghstack-poisoned] --- build/executorch-config.cmake | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build/executorch-config.cmake b/build/executorch-config.cmake index 1a2da26416e..8f64c502b47 100644 --- a/build/executorch-config.cmake +++ b/build/executorch-config.cmake @@ -118,6 +118,8 @@ endforeach() # TODO: investigate use of install(EXPORT) to cleanly handle # target_compile_options/target_compile_definitions for everything. -set_target_properties( - cpublas PROPERTIES INTERFACE_LINK_LIBRARIES extension_parallel -) +if (TARGET cpublas) + set_target_properties( + cpublas PROPERTIES INTERFACE_LINK_LIBRARIES extension_parallel + ) +endif() From 3bd64370f6454f9523b6cc51d05d377fed8a77fb Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 4 Mar 2025 19:54:10 -0800 Subject: [PATCH 8/9] Update [ghstack-poisoned] --- build/executorch-config.cmake | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build/executorch-config.cmake b/build/executorch-config.cmake index 8f64c502b47..2c459b66ac8 100644 --- a/build/executorch-config.cmake +++ b/build/executorch-config.cmake @@ -118,7 +118,12 @@ endforeach() # TODO: investigate use of install(EXPORT) to cleanly handle # target_compile_options/target_compile_definitions for everything. -if (TARGET cpublas) +if(TARGET extension_parallel) + set_target_properties( + extension_parallel PROPERTIES INTERFACE_LINK_LIBRARIES extension_threadpool + ) +endif() +if(TARGET cpublas) set_target_properties( cpublas PROPERTIES INTERFACE_LINK_LIBRARIES extension_parallel ) From 9fdebee5a3e9a439895df57b49343816fcceee86 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 4 Mar 2025 21:11:25 -0800 Subject: [PATCH 9/9] Update [ghstack-poisoned] --- build/cmake_deps.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/build/cmake_deps.toml b/build/cmake_deps.toml index 4bbfd636a96..4811563269c 100644 --- a/build/cmake_deps.toml +++ b/build/cmake_deps.toml @@ -449,6 +449,7 @@ deps = [ "executorch_core", "extension_data_loader", "extension_module", + "extension_parallel", "portable_kernels", "quantized_kernels", "xnnpack_backend",