add test

Author: lxy-9602

include/paimon/data/blob.h

@@ -94,12 +94,10 @@ class PAIMON_EXPORT Blob {
     /// It automatically injects Paimon-specific metadata to identify the field as a BLOB.
     ///
     /// @param field_name The name of the Arrow field.
-    /// @param nullable Whether the field can contain null values (defaults to false).
     /// @param metadata A map of key-value metadata to be attached to the field.
     /// @return A result containing a unique pointer to the generated `ArrowSchema` or an error.
     static Result<std::unique_ptr<::ArrowSchema>> ArrowField(
-        const std::string& field_name, bool nullable = false,
-        std::unordered_map<std::string, std::string> metadata = {});
+        const std::string& field_name, std::unordered_map<std::string, std::string> metadata = {});
 
  private:
     class Impl;

src/paimon/common/data/abstract_binary_writer.cpp

@@ -163,6 +163,8 @@ void AbstractBinaryWriter::WriteSegmentsToVarLenPart(int32_t pos,
     if (segments.size() == 1) {
         segments[0].CopyTo(offset, &segment_, cursor_, size);
     } else {
+        assert(false);
+        // Now BinarySecyion only in single segment.
         WriteMultiSegmentsToVarLenPart(segments, offset, size);
     }
     SetOffsetAndSize(pos, cursor_, size);
@@ -172,6 +174,8 @@ void AbstractBinaryWriter::WriteSegmentsToVarLenPart(int32_t pos,
 
 void AbstractBinaryWriter::WriteMultiSegmentsToVarLenPart(
     const std::vector<MemorySegment>& segments, int32_t offset, int32_t size) {
+    // Now BinarySecyion only in single segment.
+    assert(false);
     // Write the bytes to the variable length portion.
     int32_t need_copy = size;
     int32_t from_offset = offset;

src/paimon/common/data/binary_array_test.cpp

@@ -183,6 +183,7 @@ TEST(BinaryArrayTest, TestSetAndGet) {
     }
     // timestamp
     {
+        // not compact (precision > 3)
         std::vector<Timestamp> arr = {Timestamp(0, 0), Timestamp(12345, 1)};
         BinaryArray array;
         BinaryArrayWriter writer = BinaryArrayWriter(&array, arr.size(), 8, pool.get());
@@ -193,6 +194,18 @@ TEST(BinaryArrayTest, TestSetAndGet) {
         ASSERT_EQ(arr[0], array.GetTimestamp(0, 9));
         ASSERT_EQ(arr[1], array.GetTimestamp(1, 9));
     }
+    {
+        // compact (precision <= 3)
+        std::vector<Timestamp> arr = {Timestamp(0, 0), Timestamp(12345, 0)};
+        BinaryArray array;
+        BinaryArrayWriter writer = BinaryArrayWriter(&array, arr.size(), 8, pool.get());
+        for (size_t i = 0; i < arr.size(); i++) {
+            writer.WriteTimestamp(i, arr[i], 3);
+        }
+        writer.Complete();
+        ASSERT_EQ(arr[0], array.GetTimestamp(0, 3));
+        ASSERT_EQ(arr[1], array.GetTimestamp(1, 3));
+    }
     // binary
     {
         std::vector<Bytes> arr;
@@ -379,4 +392,124 @@ TEST(BinaryArrayTest, TestReset) {
     ASSERT_EQ(arr, array.ToLongArray().value());
 }
 
+TEST(BinaryArrayTest, TestGetElementSize) {
+    ASSERT_EQ(sizeof(bool), BinaryArrayWriter::GetElementSize(arrow::Type::type::BOOL));
+    ASSERT_EQ(sizeof(int8_t), BinaryArrayWriter::GetElementSize(arrow::Type::type::INT8));
+    ASSERT_EQ(sizeof(int16_t), BinaryArrayWriter::GetElementSize(arrow::Type::type::INT16));
+    ASSERT_EQ(sizeof(int32_t), BinaryArrayWriter::GetElementSize(arrow::Type::type::INT32));
+    ASSERT_EQ(sizeof(int32_t), BinaryArrayWriter::GetElementSize(arrow::Type::type::DATE32));
+    ASSERT_EQ(sizeof(int64_t), BinaryArrayWriter::GetElementSize(arrow::Type::type::INT64));
+    ASSERT_EQ(sizeof(float), BinaryArrayWriter::GetElementSize(arrow::Type::type::FLOAT));
+    ASSERT_EQ(sizeof(double), BinaryArrayWriter::GetElementSize(arrow::Type::type::DOUBLE));
+    // default cases: variable-length types use 8 bytes (offset + length)
+    ASSERT_EQ(8, BinaryArrayWriter::GetElementSize(arrow::Type::type::STRING));
+    ASSERT_EQ(8, BinaryArrayWriter::GetElementSize(arrow::Type::type::BINARY));
+    ASSERT_EQ(8, BinaryArrayWriter::GetElementSize(arrow::Type::type::TIMESTAMP));
+    ASSERT_EQ(8, BinaryArrayWriter::GetElementSize(arrow::Type::type::DECIMAL128));
+}
+
+TEST(BinaryArrayTest, TestSetNullAtWithArrowType) {
+    auto pool = GetDefaultPool();
+
+    {
+        // BOOL
+        BinaryArray array;
+        BinaryArrayWriter writer(&array, 2, sizeof(bool), pool.get());
+        writer.WriteBoolean(0, true);
+        writer.SetNullAt(1, arrow::Type::type::BOOL);
+        writer.Complete();
+        ASSERT_FALSE(array.IsNullAt(0));
+        ASSERT_TRUE(array.GetBoolean(0));
+        ASSERT_TRUE(array.IsNullAt(1));
+    }
+    {
+        // INT8
+        BinaryArray array;
+        BinaryArrayWriter writer(&array, 2, sizeof(int8_t), pool.get());
+        writer.WriteByte(0, 42);
+        writer.SetNullAt(1, arrow::Type::type::INT8);
+        writer.Complete();
+        ASSERT_FALSE(array.IsNullAt(0));
+        ASSERT_EQ(42, array.GetByte(0));
+        ASSERT_TRUE(array.IsNullAt(1));
+    }
+    {
+        // INT16
+        BinaryArray array;
+        BinaryArrayWriter writer(&array, 2, sizeof(int16_t), pool.get());
+        writer.WriteShort(0, 1000);
+        writer.SetNullAt(1, arrow::Type::type::INT16);
+        writer.Complete();
+        ASSERT_FALSE(array.IsNullAt(0));
+        ASSERT_EQ(1000, array.GetShort(0));
+        ASSERT_TRUE(array.IsNullAt(1));
+    }
+    {
+        // INT32
+        BinaryArray array;
+        BinaryArrayWriter writer(&array, 2, sizeof(int32_t), pool.get());
+        writer.WriteInt(0, 100000);
+        writer.SetNullAt(1, arrow::Type::type::INT32);
+        writer.Complete();
+        ASSERT_FALSE(array.IsNullAt(0));
+        ASSERT_EQ(100000, array.GetInt(0));
+        ASSERT_TRUE(array.IsNullAt(1));
+    }
+    {
+        // DATE32
+        BinaryArray array;
+        BinaryArrayWriter writer(&array, 2, sizeof(int32_t), pool.get());
+        writer.WriteInt(0, 19000);
+        writer.SetNullAt(1, arrow::Type::type::DATE32);
+        writer.Complete();
+        ASSERT_FALSE(array.IsNullAt(0));
+        ASSERT_EQ(19000, array.GetDate(0));
+        ASSERT_TRUE(array.IsNullAt(1));
+    }
+    {
+        // INT64
+        BinaryArray array;
+        BinaryArrayWriter writer(&array, 2, sizeof(int64_t), pool.get());
+        writer.WriteLong(0, 123456789L);
+        writer.SetNullAt(1, arrow::Type::type::INT64);
+        writer.Complete();
+        ASSERT_FALSE(array.IsNullAt(0));
+        ASSERT_EQ(123456789L, array.GetLong(0));
+        ASSERT_TRUE(array.IsNullAt(1));
+    }
+    {
+        // FLOAT
+        BinaryArray array;
+        BinaryArrayWriter writer(&array, 2, sizeof(float), pool.get());
+        writer.WriteFloat(0, 3.14f);
+        writer.SetNullAt(1, arrow::Type::type::FLOAT);
+        writer.Complete();
+        ASSERT_FALSE(array.IsNullAt(0));
+        ASSERT_FLOAT_EQ(3.14f, array.GetFloat(0));
+        ASSERT_TRUE(array.IsNullAt(1));
+    }
+    {
+        // DOUBLE
+        BinaryArray array;
+        BinaryArrayWriter writer(&array, 2, sizeof(double), pool.get());
+        writer.WriteDouble(0, 2.718);
+        writer.SetNullAt(1, arrow::Type::type::DOUBLE);
+        writer.Complete();
+        ASSERT_FALSE(array.IsNullAt(0));
+        ASSERT_DOUBLE_EQ(2.718, array.GetDouble(0));
+        ASSERT_TRUE(array.IsNullAt(1));
+    }
+    {
+        // STRING (default path, uses 8-byte null)
+        BinaryArray array;
+        BinaryArrayWriter writer(&array, 2, 8, pool.get());
+        writer.WriteString(0, BinaryString::FromString("hello", pool.get()));
+        writer.SetNullAt(1, arrow::Type::type::STRING);
+        writer.Complete();
+        ASSERT_FALSE(array.IsNullAt(0));
+        ASSERT_EQ("hello", std::string(array.GetStringView(0)));
+        ASSERT_TRUE(array.IsNullAt(1));
+    }
+}
+
 }  // namespace paimon::test

src/paimon/common/data/blob.cpp

@@ -106,9 +106,8 @@ Result<PAIMON_UNIQUE_PTR<Bytes>> Blob::ToData(const std::shared_ptr<FileSystem>&
 }
 
 Result<std::unique_ptr<ArrowSchema>> Blob::ArrowField(
-    const std::string& field_name, bool nullable,
-    std::unordered_map<std::string, std::string> metadata) {
-    auto blob_field = BlobUtils::ToArrowField(field_name, nullable, metadata);
+    const std::string& field_name, std::unordered_map<std::string, std::string> metadata) {
+    auto blob_field = BlobUtils::ToArrowField(field_name, /*nullable=*/false, metadata);
     auto field = std::make_unique<::ArrowSchema>();
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportField(*blob_field, field.get()));
     return field;

src/paimon/common/data/blob_test.cpp

@@ -16,10 +16,11 @@
 
 #include "paimon/data/blob.h"
 
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
 #include "gtest/gtest.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/memory/memory_pool.h"
-#include "paimon/status.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
@@ -142,4 +143,43 @@ TEST_F(BlobTest, TestNewInputStreamWithDynamicLength) {
     ASSERT_EQ("cdefghijklmn", buffer);
 }
 
+TEST_F(BlobTest, TestArrowField) {
+    {
+        // basic: field name, non-nullable by default
+        ASSERT_OK_AND_ASSIGN(auto schema, Blob::ArrowField("my_blob"));
+        ASSERT_NE(schema, nullptr);
+
+        // import back to arrow::Field to verify
+        auto field_result = arrow::ImportField(schema.get());
+        ASSERT_TRUE(field_result.ok());
+        auto field = field_result.ValueUnsafe();
+
+        ASSERT_EQ(field->name(), "my_blob");
+        ASSERT_EQ(field->type()->id(), arrow::Type::LARGE_BINARY);
+        ASSERT_FALSE(field->nullable());
+        ASSERT_TRUE(field->HasMetadata());
+        auto extension_type = field->metadata()->Get("paimon.extension.type");
+        ASSERT_TRUE(extension_type.ok());
+        ASSERT_EQ(extension_type.ValueUnsafe(), "paimon.type.blob");
+    }
+    {
+        // with custom metadata
+        std::unordered_map<std::string, std::string> custom_metadata = {
+            {"custom_key", "custom_value"}};
+        ASSERT_OK_AND_ASSIGN(auto schema, Blob::ArrowField("meta_blob", custom_metadata));
+        auto field = arrow::ImportField(schema.get()).ValueUnsafe();
+        ASSERT_EQ(field->name(), "meta_blob");
+        ASSERT_FALSE(field->nullable());
+        ASSERT_TRUE(field->HasMetadata());
+        // blob extension metadata should be present
+        auto extension_type = field->metadata()->Get("paimon.extension.type");
+        ASSERT_TRUE(extension_type.ok());
+        ASSERT_EQ(extension_type.ValueUnsafe(), "paimon.type.blob");
+        // custom metadata should also be present
+        auto custom_val = field->metadata()->Get("custom_key");
+        ASSERT_TRUE(custom_val.ok());
+        ASSERT_EQ(custom_val.ValueUnsafe(), "custom_value");
+    }
+}
+
 }  // namespace paimon::test

src/paimon/common/data/columnar/columnar_row_test.cpp

@@ -386,4 +386,69 @@ TEST(ColumnarRowTest, TestDataLifeCycle) {
     ASSERT_EQ(result_row->GetLong(3), 5);
 }
 
+TEST(ColumnarRowTest, TestColumnarRowRefGetBinary) {
+    auto pool = GetDefaultPool();
+    std::shared_ptr<arrow::DataType> target_type = arrow::struct_({
+        arrow::field("f0", arrow::binary()),
+        arrow::field("f1", arrow::binary()),
+    });
+    auto f0 =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::binary(), R"(["hello", "world", null])")
+            .ValueOrDie();
+    auto f1 = arrow::ipc::internal::json::ArrayFromJSON(arrow::binary(), R"(["abc", "", "xyz"])")
+                  .ValueOrDie();
+    auto data = arrow::StructArray::Make({f0, f1}, target_type->fields()).ValueOrDie();
+
+    auto ctx = std::make_shared<ColumnarBatchContext>(data->fields(), pool);
+
+    {
+        ColumnarRowRef row(ctx, 0);
+        auto binary = row.GetBinary(0);
+        ASSERT_TRUE(binary);
+        ASSERT_EQ(std::string(binary->data(), binary->size()), "hello");
+
+        auto binary1 = row.GetBinary(1);
+        ASSERT_TRUE(binary1);
+        ASSERT_EQ(std::string(binary1->data(), binary1->size()), "abc");
+    }
+    {
+        ColumnarRowRef row(ctx, 1);
+        auto binary = row.GetBinary(0);
+        ASSERT_TRUE(binary);
+        ASSERT_EQ(std::string(binary->data(), binary->size()), "world");
+
+        auto binary1 = row.GetBinary(1);
+        ASSERT_TRUE(binary1);
+        ASSERT_EQ(binary1->size(), 0);
+    }
+    {
+        ColumnarRowRef row(ctx, 2);
+        ASSERT_TRUE(row.IsNullAt(0));
+
+        auto binary1 = row.GetBinary(1);
+        ASSERT_TRUE(binary1);
+        ASSERT_EQ(std::string(binary1->data(), binary1->size()), "xyz");
+    }
+}
+
+TEST(ColumnarRowTest, TestColumnarRowRefToString) {
+    auto pool = GetDefaultPool();
+    std::shared_ptr<arrow::DataType> target_type =
+        arrow::struct_({arrow::field("f0", arrow::int32())});
+    auto f0 =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([1, 2, 3])").ValueOrDie();
+    auto data = arrow::StructArray::Make({f0}, target_type->fields()).ValueOrDie();
+
+    auto ctx = std::make_shared<ColumnarBatchContext>(data->fields(), pool);
+
+    {
+        ColumnarRowRef row(ctx, 0);
+        ASSERT_EQ(row.ToString(), "ColumnarRowRef, row_id 0");
+    }
+    {
+        ColumnarRowRef row(ctx, 2);
+        ASSERT_EQ(row.ToString(), "ColumnarRowRef, row_id 2");
+    }
+}
+
 }  // namespace paimon::test

src/paimon/common/data/data_define.h

@@ -128,18 +128,12 @@ class DataDefine {
             case arrow::Type::type::DOUBLE:
                 return Literal(GetVariantValue<double>(value));
             case arrow::Type::type::STRING: {
-                auto binary_string_ptr = GetVariantPtr<BinaryString>(value);
-                if (binary_string_ptr == nullptr) {
-                    return Status::Invalid(
-                        "VariantValueToLiteral failed, cannot get BinaryString from VariantType, "
-                        "input value maybe string view");
-                }
-                auto str = binary_string_ptr->ToString();
-                return Literal(FieldType::STRING, str.data(), str.size());
+                auto view = GetStringView(value);
+                return Literal(FieldType::STRING, view.data(), view.size());
             }
             case arrow::Type::type::BINARY: {
-                auto bytes = GetVariantValue<std::shared_ptr<Bytes>>(value);
-                return Literal(FieldType::BINARY, bytes->data(), bytes->size());
+                auto view = GetStringView(value);
+                return Literal(FieldType::BINARY, view.data(), view.size());
             }
             case arrow::Type::type::TIMESTAMP:
                 return Literal(GetVariantValue<Timestamp>(value));