feat: Add ArrowArray and ArrowArrayStream C++ iterators (#404)

All of this *should* be c++11

```diff
-  auto arrow_array = ImportArray(&array, large_utf8());
-  ARROW_EXPECT_OK(arrow_array);
-
-  auto builder = LargeStringBuilder();
-  ARROW_EXPECT_OK(builder.Append("1234"));
-  ARROW_EXPECT_OK(builder.AppendNulls(2));
-  ARROW_EXPECT_OK(builder.Append("56789"));
-  ARROW_EXPECT_OK(builder.AppendEmptyValue());
-  auto expected_array = builder.Finish();
-
-  EXPECT_TRUE(arrow_array.ValueUnsafe()->Equals(expected_array.ValueUnsafe()));
+  EXPECT_THAT(nanoarrow::ViewAsBytes<64>(&array),
+              ElementsAre("1234"_v, kNull, kNull, "56789"_v, ""_v));
```

fixes #403

---------

Co-authored-by: Dewey Dunnington <[email protected]>

Author: bkietz

.gitignore

@@ -30,3 +30,5 @@ __pycache__
 
 # meson subprojects
 subprojects
+
+compile_commands.json

CMakeLists.txt

@@ -91,6 +91,12 @@ if(NANOARROW_BUNDLE)
   file(READ src/nanoarrow/nanoarrow_testing.hpp SRC_FILE_CONTENTS)
   file(WRITE ${NANOARROW_TESTING_HPP_TEMP} "${SRC_FILE_CONTENTS}")
 
+  # Copy nanoarrow_gtest_util.hpp
+  set(NANOARROW_GTEST_UTIL_HPP_TEMP
+      ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/nanoarrow_gtest_util.hpp)
+  file(READ src/nanoarrow/nanoarrow_gtest_util.hpp SRC_FILE_CONTENTS)
+  file(WRITE ${NANOARROW_GTEST_UTIL_HPP_TEMP} "${SRC_FILE_CONTENTS}")
+
   # Combine all source files into amalgamation/nanoarrow.c in the build directory
   if(NANOARROW_BUNDLE_AS_CPP)
     set(NANOARROW_C_TEMP ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/nanoarrow.cc)
@@ -116,8 +122,12 @@ if(NANOARROW_BUNDLE)
   endif()
 
   # Install the amalgamated header and source
-  install(FILES ${NANOARROW_H_TEMP} ${NANOARROW_C_TEMP} ${NANOARROW_HPP_TEMP}
-                ${NANOARROW_TESTING_HPP_TEMP} DESTINATION ".")
+  install(FILES ${NANOARROW_H_TEMP}
+                ${NANOARROW_C_TEMP}
+                ${NANOARROW_HPP_TEMP}
+                ${NANOARROW_GTEST_UTIL_HPP_TEMP}
+                ${NANOARROW_TESTING_HPP_TEMP}
+          DESTINATION ".")
 else()
   add_library(nanoarrow src/nanoarrow/array.c src/nanoarrow/schema.c
                         src/nanoarrow/array_stream.c src/nanoarrow/utils.c)
@@ -259,12 +269,25 @@ if(NANOARROW_BUILD_TESTS)
     target_link_options(nanoarrow PUBLIC --coverage)
   endif()
 
-  target_link_libraries(utils_test nanoarrow gtest_main ${NANOARROW_ARROW_TARGET})
+  target_link_libraries(utils_test
+                        nanoarrow
+                        gtest_main
+                        gmock_main
+                        ${NANOARROW_ARROW_TARGET}
+                        nlohmann_json::nlohmann_json)
   target_link_libraries(buffer_test nanoarrow gtest_main)
-  target_link_libraries(array_test nanoarrow gtest_main ${NANOARROW_ARROW_TARGET})
+  target_link_libraries(array_test
+                        nanoarrow
+                        gtest_main
+                        gmock_main
+                        ${NANOARROW_ARROW_TARGET})
   target_link_libraries(schema_test nanoarrow gtest_main ${NANOARROW_ARROW_TARGET})
-  target_link_libraries(array_stream_test nanoarrow gtest_main)
-  target_link_libraries(nanoarrow_hpp_test nanoarrow gtest_main)
+  target_link_libraries(array_stream_test nanoarrow gtest_main gmock_main)
+  target_link_libraries(nanoarrow_hpp_test
+                        nanoarrow
+                        gtest_main
+                        gmock_main
+                        nlohmann_json::nlohmann_json)
   target_link_libraries(nanoarrow_testing_test nanoarrow gtest_main
                         nlohmann_json::nlohmann_json)
   target_link_libraries(c_data_integration_test nanoarrow nanoarrow_c_data_integration

src/nanoarrow/array_stream_test.cc

@@ -15,9 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <gmock/gmock-matchers.h>
 #include <gtest/gtest.h>
 
-#include "nanoarrow/nanoarrow.h"
+#include "nanoarrow/nanoarrow.hpp"
+
+using testing::ElementsAre;
 
 TEST(ArrayStreamTest, ArrayStreamTestBasic) {
   struct ArrowArrayStream array_stream;
@@ -44,16 +47,14 @@ TEST(ArrayStreamTest, ArrayStreamTestBasic) {
   EXPECT_STREQ(schema_copy.format, "i");
   ArrowSchemaRelease(&schema_copy);
 
-  struct ArrowArray array_copy;
-  EXPECT_EQ(ArrowArrayStreamGetNext(&array_stream, &array_copy, nullptr), NANOARROW_OK);
-  EXPECT_EQ(array_copy.length, 1);
-  EXPECT_EQ(array_copy.n_buffers, 2);
-  ArrowArrayRelease(&array_copy);
-
-  EXPECT_EQ(ArrowArrayStreamGetNext(&array_stream, &array_copy, nullptr), NANOARROW_OK);
-  EXPECT_EQ(array_copy.release, nullptr);
-
-  EXPECT_EQ(array_stream.get_last_error(&array_stream), nullptr);
+  nanoarrow::ViewArrayStream array_stream_view(&array_stream);
+  for (ArrowArray& array : array_stream_view) {
+    EXPECT_THAT(nanoarrow::ViewArrayAs<int32_t>(&array), ElementsAre(123));
+    EXPECT_EQ(array.n_buffers, 2);
+  }
+  EXPECT_EQ(array_stream_view.count(), 1);
+  EXPECT_EQ(array_stream_view.code(), NANOARROW_OK);
+  EXPECT_STREQ(array_stream_view.error()->message, "");
 
   ArrowArrayStreamRelease(&array_stream);
   EXPECT_EQ(array_stream.release, nullptr);

src/nanoarrow/array_test.cc

@@ -15,8 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <gmock/gmock-matchers.h>
 #include <gtest/gtest.h>
 #include <cmath>
+#include <cstdint>
 
 #include <arrow/array.h>
 #include <arrow/array/builder_binary.h>
@@ -29,9 +31,11 @@
 #include <arrow/compare.h>
 #include <arrow/util/decimal.h>
 
-#include "nanoarrow/nanoarrow.h"
+#include "nanoarrow/nanoarrow.hpp"
 
 using namespace arrow;
+using nanoarrow::NA;
+using testing::ElementsAre;
 
 // Lightweight versions of ArrowTesting's ARROW_EXPECT_OK. This
 // version accomplishes the task of making sure the status message
@@ -603,20 +607,9 @@ TEST(ArrayTest, ArrayTestAppendToUInt32Array) {
 
   EXPECT_EQ(array.length, 2);
   EXPECT_EQ(array.null_count, 0);
-  auto data_buffer = reinterpret_cast<const uint32_t*>(array.buffers[1]);
   EXPECT_EQ(array.buffers[0], nullptr);
-  EXPECT_EQ(data_buffer[0], 1);
-  EXPECT_EQ(data_buffer[1], 3);
-
-  auto arrow_array = ImportArray(&array, uint32());
-  ARROW_EXPECT_OK(arrow_array);
-
-  auto builder = UInt32Builder();
-  ARROW_EXPECT_OK(builder.Append(1));
-  ARROW_EXPECT_OK(builder.Append(3));
-  auto expected_array = builder.Finish();
-
-  EXPECT_TRUE(arrow_array.ValueUnsafe()->Equals(expected_array.ValueUnsafe()));
+  EXPECT_THAT(nanoarrow::ViewArrayAs<uint32_t>(&array), ElementsAre(1, 3));
+  ArrowArrayRelease(&array);
 }
 
 TEST(ArrayTest, ArrayTestAppendToUInt16Array) {
@@ -806,24 +799,8 @@ TEST(ArrayTest, ArrayTestAppendToBoolArray) {
 
   EXPECT_EQ(array.length, 4);
   EXPECT_EQ(array.null_count, 2);
-  auto validity_buffer = reinterpret_cast<const uint8_t*>(array.buffers[0]);
-  auto data_buffer = reinterpret_cast<const uint8_t*>(array.buffers[1]);
-  EXPECT_EQ(validity_buffer[0], 0x01 | 0x08);
-  EXPECT_EQ(ArrowBitGet(data_buffer, 0), 0x01);
-  EXPECT_EQ(ArrowBitGet(data_buffer, 1), 0x00);
-  EXPECT_EQ(ArrowBitGet(data_buffer, 2), 0x00);
-  EXPECT_EQ(ArrowBitGet(data_buffer, 3), 0x00);
-
-  auto arrow_array = ImportArray(&array, boolean());
-  ARROW_EXPECT_OK(arrow_array);
-
-  auto builder = BooleanBuilder();
-  ARROW_EXPECT_OK(builder.Append(true));
-  ARROW_EXPECT_OK(builder.AppendNulls(2));
-  ARROW_EXPECT_OK(builder.Append(false));
-  auto expected_array = builder.Finish();
-
-  EXPECT_TRUE(arrow_array.ValueUnsafe()->Equals(expected_array.ValueUnsafe()));
+  EXPECT_THAT(nanoarrow::ViewArrayAs<bool>(&array), ElementsAre(true, NA, NA, false));
+  ArrowArrayRelease(&array);
 }
 
 TEST(ArrayTest, ArrayTestAppendToLargeStringArray) {
@@ -855,17 +832,9 @@ TEST(ArrayTest, ArrayTestAppendToLargeStringArray) {
   EXPECT_EQ(offset_buffer[4], 9);
   EXPECT_EQ(memcmp(data_buffer, "123456789", 9), 0);
 
-  auto arrow_array = ImportArray(&array, large_utf8());
-  ARROW_EXPECT_OK(arrow_array);
-
-  auto builder = LargeStringBuilder();
-  ARROW_EXPECT_OK(builder.Append("1234"));
-  ARROW_EXPECT_OK(builder.AppendNulls(2));
-  ARROW_EXPECT_OK(builder.Append("56789"));
-  ARROW_EXPECT_OK(builder.AppendEmptyValue());
-  auto expected_array = builder.Finish();
-
-  EXPECT_TRUE(arrow_array.ValueUnsafe()->Equals(expected_array.ValueUnsafe()));
+  EXPECT_THAT(nanoarrow::ViewArrayAsBytes<64>(&array),
+              ElementsAre("1234"_v, NA, NA, "56789"_v, ""_v));
+  ArrowArrayRelease(&array);
 }
 
 TEST(ArrayTest, ArrayTestAppendToFixedSizeBinaryArray) {
@@ -899,18 +868,10 @@ TEST(ArrayTest, ArrayTestAppendToFixedSizeBinaryArray) {
                           '9',  '0',  0x00, 0x00, 0x00, 0x00, 0x00};
   EXPECT_EQ(memcmp(data_buffer, expected_data, 25), 0);
 
-  auto arrow_array = ImportArray(&array, &schema);
-  ARROW_EXPECT_OK(arrow_array);
-
-  auto builder = FixedSizeBinaryBuilder(fixed_size_binary(5));
-  ARROW_EXPECT_OK(builder.Append("12345"));
-  ARROW_EXPECT_OK(builder.AppendNulls(2));
-  ARROW_EXPECT_OK(builder.Append("67890"));
-  ARROW_EXPECT_OK(builder.AppendEmptyValue());
-  auto expected_array = builder.Finish();
-  ARROW_EXPECT_OK(expected_array);
-
-  EXPECT_TRUE(arrow_array.ValueUnsafe()->Equals(expected_array.ValueUnsafe()));
+  EXPECT_THAT(nanoarrow::ViewArrayAsFixedSizeBytes(&array, 5),
+              ElementsAre("12345"_v, NA, NA, "67890"_v, "\0\0\0\0\0"_v));
+  ArrowArrayRelease(&array);
+  ArrowSchemaRelease(&schema);
 }
 
 TEST(ArrayTest, ArrayTestAppendToBinaryArrayErrors) {
@@ -932,7 +893,7 @@ TEST(ArrayTest, ArrayTestAppendToIntervalArrayYearMonth) {
   const int32_t months = 42;
   struct ArrowInterval interval;
   ArrowIntervalInit(&interval, ArrowType::NANOARROW_TYPE_INTERVAL_MONTHS);
-  interval.months = 42;
+  interval.months = months;
 
   ASSERT_EQ(ArrowArrayInitFromType(&array, NANOARROW_TYPE_INTERVAL_MONTHS), NANOARROW_OK);
   EXPECT_EQ(ArrowArrayStartAppending(&array), NANOARROW_OK);
@@ -943,14 +904,8 @@ TEST(ArrayTest, ArrayTestAppendToIntervalArrayYearMonth) {
   EXPECT_EQ(array.length, 2);
   EXPECT_EQ(array.null_count, 1);
 
-  auto data_buffer = reinterpret_cast<const int32_t*>(array.buffers[1]);
-  EXPECT_EQ(data_buffer[0], months);
-
-  auto arrow_array = ImportArray(&array, month_interval());
-  ARROW_EXPECT_OK(arrow_array);
-
-  // TODO: arrow does not have a builder for MonthIntervals
-  // so no comparison is done after creating the array
+  EXPECT_THAT(nanoarrow::ViewArrayAs<int32_t>(&array), ElementsAre(months, NA));
+  ArrowArrayRelease(&array);
 }
 
 TEST(ArrayTest, ArrayTestAppendToIntervalArrayDayTime) {