diff --git a/presto-docs/src/main/sphinx/presto-cpp.rst b/presto-docs/src/main/sphinx/presto-cpp.rst index 894fab1767f34..79e0c3fc13263 100644 --- a/presto-docs/src/main/sphinx/presto-cpp.rst +++ b/presto-docs/src/main/sphinx/presto-cpp.rst @@ -9,6 +9,7 @@ Note: Presto C++ is in active development. See :doc:`Limitations + struct nameOfStruct { + FOLLY_ALWAYS_INLINE bool call(int64_t& result) { + ... + } + }; + + extern "C" { + void registry() { + facebook::presto::registerPrestoFunction< + nameOfStruct, + int64_t>("function_name"); + } + } + +Note: the int64_t return type can be changed as needed. For more examples, See the `README `_ + +2. Create a shared library which may be made using CMakeLists like the following: + +.. code-block:: text + + add_library(name_of_dynamic_fn SHARED TestFunction.cpp) + target_link_libraries(name_of_dynamic_fn PRIVATE fmt::fmt Folly::folly gflags::gflags) + +3. `Drop your shared libraries into the plugin directory <../plugin.rst>`_ \ No newline at end of file diff --git a/presto-native-execution/presto_cpp/main/CMakeLists.txt b/presto-native-execution/presto_cpp/main/CMakeLists.txt index c06e00edf834c..e646fcce39f3e 100644 --- a/presto-native-execution/presto_cpp/main/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/CMakeLists.txt @@ -14,6 +14,7 @@ add_subdirectory(types) add_subdirectory(http) add_subdirectory(common) add_subdirectory(thrift) +add_subdirectory(dynamic_registry) add_library( presto_server_lib @@ -62,9 +63,11 @@ target_link_libraries( velox_dwio_orc_reader velox_dwio_parquet_reader velox_dwio_parquet_writer + velox_dynamic_library_loader velox_encode velox_exec velox_file + velox_function_registry velox_functions_lib velox_functions_prestosql velox_gcs @@ -100,6 +103,15 @@ set_property(TARGET presto_server_lib PROPERTY JOB_POOL_LINK add_executable(presto_server PrestoMain.cpp) +# The below additional flags are necessary for resolving dependencies for +# loading dynamic libraries. +if(APPLE) + target_link_options(presto_server BEFORE PUBLIC + "-Wl,-undefined,dynamic_lookup") +else() + target_link_options(presto_server BEFORE PUBLIC "-Wl,-export-dynamic") +endif() + # Moving velox_hive_connector and velox_tpch_connector to presto_server_lib # results in multiple link errors similar to the one below only on GCC. # "undefined reference to `vtable for velox::connector::tpch::TpchTableHandle`" diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.cpp b/presto-native-execution/presto_cpp/main/PrestoServer.cpp index 0a422c66e16bd..1fff98a567c18 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoServer.cpp @@ -44,6 +44,7 @@ #include "velox/common/base/StatsReporter.h" #include "velox/common/caching/CacheTTLController.h" #include "velox/common/caching/SsdCache.h" +#include "velox/common/dynamic_registry/DynamicLibraryLoader.h" #include "velox/common/file/FileSystems.h" #include "velox/common/memory/MmapAllocator.h" #include "velox/common/memory/SharedArbitrator.h" @@ -396,6 +397,7 @@ void PrestoServer::run() { registerRemoteFunctions(); registerVectorSerdes(); registerPrestoPlanNodeSerDe(); + registerDynamicFunctions(); const auto numExchangeHttpClientIoThreads = std::max( systemConfig->exchangeHttpClientNumIoThreadsHwMultiplier() * @@ -1598,5 +1600,28 @@ protocol::NodeStatus PrestoServer::fetchNodeStatus() { return nodeStatus; } +void PrestoServer::registerDynamicFunctions() { + auto systemConfig = SystemConfig::instance(); + if (!systemConfig->pluginDir().empty()) { + // If user provided path is valid, traverse and call dynamic function loader + // for all shared libraries. + const fs::path path(systemConfig->pluginDir()); + PRESTO_STARTUP_LOG(INFO) << path; + std::error_code ec; + if (fs::is_directory(path, ec)) { + using recursiveDirectoryIterator = + std::filesystem::recursive_directory_iterator; + std::set extensions{".so", ".dylib"}; + for (const auto& dirEntry : recursiveDirectoryIterator(path)) { + // Skip any non shared library files and directories from loading. + auto dirEntryPath = dirEntry.path(); + if (!fs::is_directory(dirEntry, ec) && + extensions.find(dirEntryPath.extension()) != extensions.end()) { + velox::loadDynamicLibrary(dirEntryPath.c_str()); + } + } + } + } +} } // namespace facebook::presto diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.h b/presto-native-execution/presto_cpp/main/PrestoServer.h index 9fa1301c1bf1d..df6d6f2c76c8b 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.h +++ b/presto-native-execution/presto_cpp/main/PrestoServer.h @@ -190,6 +190,8 @@ class PrestoServer { VeloxPlanValidator* getVeloxPlanValidator(); + virtual void registerDynamicFunctions(); + /// Invoked to get the list of filters passed to the http server. std::vector> getHttpServerFilters(); diff --git a/presto-native-execution/presto_cpp/main/common/Configs.cpp b/presto-native-execution/presto_cpp/main/common/Configs.cpp index 2ef3d8253ab3d..d22ecc51e142b 100644 --- a/presto-native-execution/presto_cpp/main/common/Configs.cpp +++ b/presto-native-execution/presto_cpp/main/common/Configs.cpp @@ -240,6 +240,7 @@ SystemConfig::SystemConfig() { BOOL_PROP(kEnableRuntimeMetricsCollection, false), BOOL_PROP(kPlanValidatorFailOnNestedLoopJoin, false), STR_PROP(kPrestoDefaultNamespacePrefix, "presto.default"), + STR_PROP(kPluginDir, ""), }; } @@ -763,6 +764,10 @@ std::string SystemConfig::prestoDefaultNamespacePrefix() const { return optionalProperty(kPrestoDefaultNamespacePrefix).value().append("."); } +std::string SystemConfig::pluginDir() const { + return optionalProperty(kPluginDir).value(); +} + NodeConfig::NodeConfig() { registeredProps_ = std::unordered_map>{ diff --git a/presto-native-execution/presto_cpp/main/common/Configs.h b/presto-native-execution/presto_cpp/main/common/Configs.h index 5969006e62575..a43aa398d85e2 100644 --- a/presto-native-execution/presto_cpp/main/common/Configs.h +++ b/presto-native-execution/presto_cpp/main/common/Configs.h @@ -642,6 +642,8 @@ class SystemConfig : public ConfigBase { static constexpr std::string_view kInternalCommunicationJwtExpirationSeconds{ "internal-communication.jwt.expiration-seconds"}; + /// Optional string containing the path to the plugin directory + static constexpr std::string_view kPluginDir{"plugin.dir"}; /// Below are the Presto properties from config.properties that get converted /// to their velox counterparts in BaseVeloxQueryConfig and used solely from /// BaseVeloxQueryConfig. @@ -898,6 +900,8 @@ class SystemConfig : public ConfigBase { bool prestoNativeSidecar() const; std::string prestoDefaultNamespacePrefix() const; + + std::string pluginDir() const; }; /// Provides access to node properties defined in node.properties file. diff --git a/presto-native-execution/presto_cpp/main/dynamic_registry/DynamicFunctionRegistrar.h b/presto-native-execution/presto_cpp/main/dynamic_registry/DynamicFunctionRegistrar.h new file mode 100644 index 0000000000000..5a38d454fc30c --- /dev/null +++ b/presto-native-execution/presto_cpp/main/dynamic_registry/DynamicFunctionRegistrar.h @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "presto_cpp/main/common/Configs.h" +#include "velox/functions/Macros.h" +#include "velox/functions/Registerer.h" +namespace facebook::presto { +template