chore: add test workflows for gcc

.github/workflows/test.yaml → .github/workflows/clang_test.yaml

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: Test
+name: Clang Test
 
 on:
   push:
@@ -49,4 +49,4 @@ jobs:
         env:
           CC: clang
           CXX: clang++
-        run: ci/scripts/build_paimon.sh $(pwd) ON
+        run: ci/scripts/build_paimon.sh $(pwd)

.github/workflows/gcc_test.yaml

@@ -0,0 +1,52 @@
+# Copyright 2024-present Alibaba Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Gcc Test
+
+on:
+  push:
+    branches:
+      - '**'
+    tags:
+      - '**'
+  pull_request:
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  PAIMON_HOME: /tmp/paimon
+
+jobs:
+  ubuntu:
+    name: AMD64 Ubuntu 24.04
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Checkout paimon-cpp
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          lfs: true
+      - name: Build Paimon
+        shell: bash
+        env:
+          CC: gcc-14
+          CXX: g++-14
+        run: ci/scripts/build_paimon.sh $(pwd)

.github/workflows/test_with_sanitizer.yaml

@@ -0,0 +1,52 @@
+# Copyright 2024-present Alibaba Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Test with sanitizer
+
+on:
+  push:
+    branches:
+      - '**'
+    tags:
+      - '**'
+  pull_request:
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  PAIMON_HOME: /tmp/paimon
+
+jobs:
+  ubuntu:
+    name: AMD64 Ubuntu 24.04
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Checkout paimon-cpp
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          lfs: true
+      - name: Build Paimon
+        shell: bash
+        env:
+          CC: clang
+          CXX: clang++
+        run: ci/scripts/build_paimon.sh $(pwd) true

CMakeLists.txt

@@ -46,7 +46,6 @@ option(PAIMON_BUILD_STATIC "Build static library" ON)
 option(PAIMON_BUILD_SHARED "Build shared library" ON)
 option(PAIMON_BUILD_TESTS "Build tests" OFF)
 option(PAIMON_USE_ASAN "Use Address Sanitizer" OFF)
-option(PAIMON_USE_TSAN "Use Thread Sanitizer" OFF)
 option(PAIMON_USE_UBSAN "Use Undefined Behavior Sanitizer" OFF)
 option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
 option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
@@ -331,23 +330,35 @@ if(PAIMON_BUILD_TESTS)
         paimon_global_index_static
         paimon_local_file_system_static
         paimon_mock_file_format_static
+        -Wl,--no-as-needed
         paimon_parquet_file_format_shared
-        paimon_blob_file_format_shared)
+        paimon_blob_file_format_shared
+        -Wl,--as-needed)
 
     if(PAIMON_ENABLE_LANCE)
+        list(APPEND TEST_STATIC_LINK_LIBS -Wl,--no-as-needed)
         list(APPEND TEST_STATIC_LINK_LIBS paimon_lance_file_format_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS -Wl,--as-needed)
     endif()
     if(PAIMON_ENABLE_ORC)
+        list(APPEND TEST_STATIC_LINK_LIBS -Wl,--no-as-needed)
         list(APPEND TEST_STATIC_LINK_LIBS paimon_orc_file_format_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS -Wl,--as-needed)
     endif()
     if(PAIMON_ENABLE_AVRO)
+        list(APPEND TEST_STATIC_LINK_LIBS -Wl,--no-as-needed)
         list(APPEND TEST_STATIC_LINK_LIBS paimon_avro_file_format_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS -Wl,--as-needed)
     endif()
     if(PAIMON_ENABLE_JINDO)
+        list(APPEND TEST_STATIC_LINK_LIBS -Wl,--no-as-needed)
         list(APPEND TEST_STATIC_LINK_LIBS paimon_jindo_file_system_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS -Wl,--as-needed)
     endif()
     if(PAIMON_ENABLE_LUMINA)
+        list(APPEND TEST_STATIC_LINK_LIBS -Wl,--no-as-needed)
         list(APPEND TEST_STATIC_LINK_LIBS paimon_lumina_index_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS -Wl,--as-needed)
     endif()
 
 endif()

ci/scripts/build_paimon.sh

@@ -17,6 +17,7 @@
 set -eux
 
 source_dir=${1}
+enable_sanitizer=${2:-false}
 build_dir=${1}/build
 
 mkdir ${build_dir}
@@ -30,6 +31,13 @@ CMAKE_ARGS=(
     "-DPAIMON_ENABLE_JINDO=ON"
 )
 
+if [[ "${enable_sanitizer}" == "true" ]]; then
+    CMAKE_ARGS+=(
+        "-DPAIMON_USE_ASAN=ON"
+        "-DPAIMON_USE_UBSAN=ON"
+    )
+fi
+
 cmake "${CMAKE_ARGS[@]}" ${source_dir}
 cmake --build . -- -j$(nproc)
 ctest --output-on-failure -j $(nproc)

cmake_modules/san-config.cmake

@@ -10,11 +10,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License. See accompanying LICENSE file.
 
-# Clang does not support using ASAN and TSAN simultaneously.
-if("${PAIMON_USE_ASAN}" AND "${PAIMON_USE_TSAN}")
-    message(SEND_ERROR "Can only enable one of ASAN or TSAN at a time")
-endif()
-
 add_library(paimon_sanitizer_flags INTERFACE)
 
 if(PAIMON_USE_ASAN)
@@ -28,17 +23,6 @@ if(PAIMON_USE_ASAN)
     endif()
 endif()
 
-if(PAIMON_USE_TSAN)
-    if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
-        target_compile_options(paimon_sanitizer_flags INTERFACE -fsanitize=thread
-                                                                -fno-omit-frame-pointer)
-        target_link_options(paimon_sanitizer_flags INTERFACE -fsanitize=thread)
-        message(STATUS "Thread Sanitizer enabled")
-    else()
-        message(WARNING "Thread Sanitizer is only supported for GCC and Clang compilers")
-    endif()
-endif()
-
 if(PAIMON_USE_UBSAN)
     if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
         target_compile_options(paimon_sanitizer_flags INTERFACE -fsanitize=undefined

src/paimon/format/orc/orc_input_output_stream_test.cpp

@@ -97,7 +97,7 @@ TEST(OrcInputOutputStreamTest, TestSimple) {
     std::unique_ptr<::orc::ColumnVectorBatch> batch = writer->createRowBatch(batch_size);
     auto* struct_batch = dynamic_cast<::orc::StructVectorBatch*>(batch.get());
     ASSERT_TRUE(struct_batch);
-    auto* int_batch = static_cast<::orc::IntVectorBatch*>(struct_batch->fields[0]);
+    auto* int_batch = dynamic_cast<::orc::LongVectorBatch*>(struct_batch->fields[0]);
     ASSERT_TRUE(int_batch);
     auto* double_batch = dynamic_cast<::orc::DoubleVectorBatch*>(struct_batch->fields[1]);
     ASSERT_TRUE(double_batch);
@@ -159,7 +159,7 @@ TEST(OrcInputOutputStreamTest, TestSimple) {
 
     struct_batch = dynamic_cast<::orc::StructVectorBatch*>(batch.get());
     ASSERT_TRUE(struct_batch);
-    int_batch = static_cast<::orc::IntVectorBatch*>(struct_batch->fields[0]);
+    int_batch = static_cast<::orc::LongVectorBatch*>(struct_batch->fields[0]);
     ASSERT_TRUE(int_batch);
     double_batch = dynamic_cast<::orc::DoubleVectorBatch*>(struct_batch->fields[1]);
     ASSERT_TRUE(double_batch);