Merge branch 'main' into fix_some_ut

Author: SGZW

.github/workflows/build_release.yaml

@@ -0,0 +1,64 @@
+# Copyright 2025-present Alibaba Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build Release
+
+on:
+  push:
+    branches:
+      - '**'
+    tags:
+      - '**'
+  pull_request:
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  clang-release:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Checkout paimon-cpp
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          lfs: true
+      - name: Build Paimon
+        shell: bash
+        env:
+          CC: clang
+          CXX: clang++
+        run: ci/scripts/build_paimon.sh $(pwd) false false Release
+  gcc-release:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Checkout paimon-cpp
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          lfs: true
+      - name: Build Paimon
+        shell: bash
+        env:
+          CC: gcc-14
+          CXX: g++-14
+        run: ci/scripts/build_paimon.sh $(pwd) false false Release

.github/workflows/clang_test.yaml

@@ -40,6 +40,7 @@ jobs:
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           lfs: true
+          fetch-depth: 0  # fetch all history for git diff in clang-tidy
       - name: Build Paimon
         shell: bash
         env:

CMakeLists.txt

@@ -323,7 +323,6 @@ if(PAIMON_BUILD_TESTS)
                       -L
                       unittest
                       --output-on-failure)
-    add_compile_options(-fno-access-control)
     add_dependencies(unittest paimon-tests)
 
     include_directories(SYSTEM ${GTEST_INCLUDE_DIR})

README.md

@@ -66,7 +66,7 @@ The writing is divided into two stages:
                                .AddOption(Options::FILE_SYSTEM, "local")
                                .IgnoreEmptyCommit(false)
                                .Finish());
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit, FileStoreCommit::Create(std::move(commit_context));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit, FileStoreCommit::Create(std::move(commit_context)));
     PAIMON_RETURN_NOT_OK(commit->Commit(commit_messages));
 
 ```
@@ -100,7 +100,7 @@ The reading is divided into two stages:
                                .AddOption(Options::FILE_SYSTEM, "local")
                                .EnablePrefetch(true)
                                .Finish());
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read, TableRead::Create(std::move(read_context));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read, TableRead::Create(std::move(read_context)));
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> batch_reader, table_read->CreateReader(plan->Splits()));
 
     while (true) {

ci/scripts/build_paimon.sh

@@ -19,14 +19,15 @@ set -eux
 source_dir=${1}
 enable_sanitizer=${2:-false}
 check_clang_tidy=${3:-false}
+build_type=${4:-Debug}
 build_dir=${1}/build
 
 mkdir ${build_dir}
 pushd ${build_dir}
 
 CMAKE_ARGS=(
     "-G Ninja"
-    "-DCMAKE_BUILD_TYPE=Debug"
+    "-DCMAKE_BUILD_TYPE=${build_type}"
     "-DPAIMON_BUILD_TESTS=ON"
     "-DPAIMON_ENABLE_LANCE=ON"
     "-DPAIMON_ENABLE_JINDO=ON"

cmake_modules/BuildUtils.cmake

@@ -256,6 +256,7 @@ function(add_test_case REL_TEST_NAME)
     if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
         target_compile_options(${TEST_NAME} PRIVATE -Wno-global-constructors)
     endif()
+    target_compile_options(${TEST_NAME} PRIVATE -fno-access-control)
 
     add_test(${TEST_NAME}
              ${BUILD_SUPPORT_DIR}/run-test.sh

cmake_modules/DefineOptions.cmake

@@ -99,7 +99,6 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     set_option_category("Test")
 
     define_option(PAIMON_BUILD_TESTS "Build the Paimon googletest unit tests" OFF)
-    define_option(PAIMON_BUILD_EXAMPLE "Build the Paimon example" OFF)
 
     if(PAIMON_BUILD_SHARED)
         set(PAIMON_TEST_LINKAGE_DEFAULT "shared")
@@ -117,7 +116,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
                   "If on, only git-diff files will be passed to linting tools" ON)
 
     define_option_string(PAIMON_LINT_GIT_TARGET_COMMIT
-                         "target commit/branch for comparison in git diff" "HEAD")
+                         "target commit/branch for comparison in git diff" "origin/main")
 
     define_option(PAIMON_GENERATE_COVERAGE "Build with C++ code coverage enabled" OFF)
 

cmake_modules/ThirdpartyToolchain.cmake

@@ -249,8 +249,12 @@ macro(build_fmt)
     string(REPLACE "-Werror" "" FMT_CMAKE_CXX_FLAGS ${FMT_CMAKE_CXX_FLAGS})
 
     set(FMT_CMAKE_ARGS
-        ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${FMT_PREFIX}
-        "-DCMAKE_CXX_FLAGS=${FMT_CMAKE_CXX_FLAGS}" "-DCMAKE_C_FLAGS=${FMT_CMAKE_C_FLAGS}")
+        ${EP_COMMON_CMAKE_ARGS}
+        -DCMAKE_INSTALL_PREFIX=${FMT_PREFIX}
+        "-DCMAKE_CXX_FLAGS=${FMT_CMAKE_CXX_FLAGS}"
+        "-DCMAKE_C_FLAGS=${FMT_CMAKE_C_FLAGS}"
+        -DFMT_TEST=OFF
+        -DFMT_DOC=OFF)
     set(FMT_CONFIGURE CMAKE_ARGS ${FMT_CMAKE_ARGS})
     externalproject_add(fmt_ep
                         URL ${FMT_SOURCE_URL}
@@ -337,7 +341,8 @@ macro(build_zstd)
         -DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX}
         "-DCMAKE_CXX_FLAGS=${ZSTD_CMAKE_CXX_FLAGS}"
         "-DCMAKE_C_FLAGS=${ZSTD_CMAKE_C_FLAGS}"
-        -DZSTD_BUILD_SHARED=OFF)
+        -DZSTD_BUILD_SHARED=OFF
+        -DZSTD_BUILD_PROGRAMS=OFF)
 
     set(ZSTD_CONFIGURE SOURCE_SUBDIR "build/cmake" CMAKE_ARGS ${ZSTD_CMAKE_ARGS})
     externalproject_add(zstd_ep
@@ -363,7 +368,8 @@ macro(build_lz4)
         "${LZ4_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lz4${CMAKE_STATIC_LIBRARY_SUFFIX}"
     )
     set(LZ4_LIBRARIES ${LZ4_STATIC_LIB})
-    set(LZ4_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${LZ4_PREFIX})
+    set(LZ4_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${LZ4_PREFIX}
+                       -DLZ4_BUILD_CLI=OFF -DLZ4_BUILD_LEGACY_LZ4C=OFF)
 
     set(LZ4_CONFIGURE SOURCE_SUBDIR "build/cmake" CMAKE_ARGS ${LZ4_CMAKE_ARGS})
     externalproject_add(lz4_ep
@@ -548,6 +554,13 @@ macro(build_avro)
 
     get_target_property(AVRO_ZSTD_INCLUDE_DIR zstd INTERFACE_INCLUDE_DIRECTORIES)
     get_filename_component(AVRO_ZSTD_ROOT "${AVRO_ZSTD_INCLUDE_DIR}" DIRECTORY)
+
+    get_target_property(AVRO_ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES)
+    get_filename_component(AVRO_ZLIB_ROOT "${AVRO_ZLIB_INCLUDE_DIR}" DIRECTORY)
+
+    get_target_property(AVRO_FMT_INCLUDE_DIR fmt INTERFACE_INCLUDE_DIRECTORIES)
+    get_filename_component(AVRO_FMT_ROOT "${AVRO_FMT_INCLUDE_DIR}" DIRECTORY)
+
     set(AVRO_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error")
     set(AVRO_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error")
 
@@ -558,7 +571,8 @@ macro(build_avro)
         "-DCMAKE_C_FLAGS=${AVRO_CMAKE_C_FLAGS}"
         "-DAVRO_BUILD_TESTS=OFF"
         "-DAVRO_BUILD_EXECUTABLES=OFF"
-        "-DZLIB_ROOT=${THIRDPARTY_ZLIB_ROOT}"
+        "-DZLIB_ROOT=${AVRO_ZLIB_ROOT}"
+        "-Dfmt_ROOT=${AVRO_FMT_ROOT}"
         "-Dzstd_ROOT=${AVRO_ZSTD_ROOT}"
         "-DSnappy_ROOT=${AVRO_SNAPPY_ROOT}")
     externalproject_add(avro_ep
@@ -567,7 +581,7 @@ macro(build_avro)
                         SOURCE_SUBDIR "lang/c++"
                         CMAKE_ARGS ${AVRO_CMAKE_ARGS}
                         BUILD_BYPRODUCTS "${AVRO_STATIC_LIB}"
-                        DEPENDS zlib zstd snappy)
+                        DEPENDS fmt zlib zstd snappy)
 
     file(MAKE_DIRECTORY "${AVRO_INCLUDE_DIR}")
 
@@ -693,6 +707,9 @@ macro(build_arrow)
     set(ARROW_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error")
     set(ARROW_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error")
     string(REPLACE "-Werror" "" ARROW_CMAKE_CXX_FLAGS ${ARROW_CMAKE_CXX_FLAGS})
+    # Fix for thrift Mutex.h missing #include <cstdint> (GCC 15 strictness)
+    # Use -include to force include cstdint for all C++ files
+    string(APPEND ARROW_CMAKE_CXX_FLAGS " -include cstdint")
 
     set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-install")
     set(ARROW_HOME "${ARROW_PREFIX}")
@@ -730,6 +747,9 @@ macro(build_arrow)
         -DARROW_DEPENDENCY_USE_SHARED=OFF
         -DARROW_BUILD_SHARED=OFF
         -DARROW_BUILD_STATIC=ON
+        -DARROW_BUILD_TESTS=OFF
+        -DARROW_BUILD_BENCHMARKS=OFF
+        -DARROW_BUILD_EXAMPLES=OFF
         -DARROW_JEMALLOC=OFF
         -DARROW_WITH_RE2=OFF
         -DARROW_WITH_UTF8PROC=OFF
@@ -988,6 +1008,11 @@ macro(build_glog)
                                      INTERFACE_COMPILE_DEFINITIONS "GLOG_USE_GLOG_EXPORT")
 
     add_dependencies(glog glog_ep)
+
+    find_library(LIBUNWIND_LIBRARY NAMES unwind)
+    if(LIBUNWIND_LIBRARY)
+        target_link_libraries(glog INTERFACE ${LIBUNWIND_LIBRARY})
+    endif()
 endmacro()
 
 build_fmt()

docs/source/api/global_index.rst

@@ -13,7 +13,7 @@
 .. limitations under the License.
 
 ============
-GLobal Index
+Global Index
 ============
 
 .. _cpp-api-global-index:

include/paimon/catalog/catalog.h

@@ -95,15 +95,26 @@ class PAIMON_EXPORT Catalog {
     /// status.
     virtual Result<std::vector<std::string>> ListTables(const std::string& db_name) const = 0;
 
+    /// Checks whether a database with the specified name exists in the catalog.
+    ///
+    /// @param db_name The name of the database to check for existence.
+    /// @return A result containing true if the database exists, false otherwise, or an error
+    /// status.
+    virtual Result<bool> DatabaseExists(const std::string& db_name) const = 0;
+
+    /// Checks whether a table with the specified identifier exists in the catalog.
+    ///
+    /// @param identifier The identifier of the table to check for existence.
+    /// @return A result containing true if the table exists, false otherwise, or an error status.
+    virtual Result<bool> TableExists(const Identifier& identifier) const = 0;
+
     /// Loads the latest schema of a specified table.
     ///
     /// @note System tables will not be supported.
     ///
     /// @param identifier The identifier (database and table name) of the table to load.
-    /// @return A result containing table schema if the table exists, or std::nullopt if it
-    /// doesn't, or an error status on failure.
-    virtual Result<std::optional<std::shared_ptr<Schema>>> LoadTableSchema(
-        const Identifier& identifier) const = 0;
+    /// @return A result containing table schema if the table exists, or an error status on failure.
+    virtual Result<std::shared_ptr<Schema>> LoadTableSchema(const Identifier& identifier) const = 0;
 };
 
 }  // namespace paimon