Skip to content

Enhancement/18232393759/update sparrow to 1.3.0 #2725

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 24, 2025
Merged

Enhancement/18232393759/update sparrow to 1.3.0 #2725

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a1f80a2
Update vcpkg baseline and sparrow to 1.3.0
alexowens90 Oct 22, 2025
4b45282
Use new offset(), null_count(), and ditionary() methods from sparrow:…
alexowens90 Oct 23, 2025
b0e2878
Add vcpkg overlay to pin ninja version used in CI (#2731)
G-D-Petrov Oct 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 4 additions & 9 deletions cpp/arcticdb/arrow/arrow_utils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,11 +270,8 @@ std::shared_ptr<std::vector<sparrow::record_batch>> segment_to_arrow_data(Segmen
}

DataType arcticdb_type_from_arrow_array(const sparrow::array& array) {
// Remove const-cast once https://github.com/man-group/sparrow/issues/587 is released
// Remove use of get_arrow_array completely when https://github.com/man-group/sparrow/issues/588 is released
schema::check<ErrorCode::E_UNSUPPORTED_COLUMN_TYPE>(
sparrow::get_arrow_array(const_cast<sparrow::array&>(array))->dictionary == nullptr,
"Dictionary-encoded Arrow data unsupported"
!array.dictionary().has_value(), "Dictionary-encoded Arrow data unsupported"
);
switch (array.data_type()) {
case sparrow::data_type::BOOL:
Expand Down Expand Up @@ -364,9 +361,8 @@ std::pair<SegmentInMemory, std::optional<size_t>> arrow_data_to_segment(
const auto& data_type = data_types[idx];
const auto& array = record_batch->get_column(idx);
auto [arrow_array, arrow_schema] = sparrow::get_arrow_structures(array);
// Remove use of arrow_array here when https://github.com/man-group/sparrow/issues/589 is released
schema::check<ErrorCode::E_UNSUPPORTED_COLUMN_TYPE>(
arrow_array->null_count == 0,
array.null_count() == 0,
"Column '{}' contains null values, which are not currently supported",
record_batch->names()[idx]
);
Expand All @@ -382,11 +378,10 @@ std::pair<SegmentInMemory, std::optional<size_t>> arrow_data_to_segment(
column.buffer() = ChunkedBuffer::presized(total_rows);
}
packed_bits_to_buffer(
data, array.size(), arrow_array->offset, column.buffer().bytes_at(start_row, array.size())
data, array.size(), array.offset(), column.buffer().bytes_at(start_row, array.size())
);
} else { // Numeric and string types
// Remove use of arrow_array here when https://github.com/man-group/sparrow/issues/589 is released
data += arrow_array->offset * get_type_size(data_type);
data += array.offset() * get_type_size(data_type);
// For string columns, we deliberately omit the last value from the offsets buffer to keep our indexing
// into the column's ChunkedBuffer accurate. See corresponding comment in
// WriteToSegmentTask::slice_column
Expand Down
56 changes: 56 additions & 0 deletions cpp/third_party/vcpkg_overlays/ninja/vcpkg_find_acquire_program(NINJA).cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
# Override ninja version to use 1.10.2 which is compatible with older GLIBCXX
set(program_name ninja)
set(program_version 1.10.2)
set(tool_subdirectory)
set(interpreter)
set(search_names ninja)
set(download_urls)
set(download_filename)
set(supported_on_unix ON)
set(version_command --version)
set(paths_to_search)
set(post_install_command)

if(CMAKE_HOST_WIN32)
set(download_filename "ninja-win-${program_version}.zip")
set(download_urls "https://github.com/ninja-build/ninja/releases/download/v${program_version}/ninja-win.zip")
set(tool_subdirectory "${program_version}-windows")
set(paths_to_search "${DOWNLOADS}/tools/ninja-${program_version}-windows")
set(download_sha512 6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a426fc1af0d0efb4d8c30f)
elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
set(download_filename "ninja-mac-${program_version}.zip")
set(download_urls "https://github.com/ninja-build/ninja/releases/download/v${program_version}/ninja-mac.zip")
set(tool_subdirectory "${program_version}-osx")
set(paths_to_search "${DOWNLOADS}/tools/ninja-${program_version}-osx")
set(download_sha512 482ecb23c59ae3d4f158029112de172f0c513bc7667ee5f74e59eac3f05a0c9fd94bd0c3a0d81aaf6ca286a16e6f5998c428193e3c49fd2a1922a7b22e2fafe9)
elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux")
vcpkg_execute_in_download_mode(COMMAND "uname" "-m" OUTPUT_VARIABLE HOST_ARCH OUTPUT_STRIP_TRAILING_WHITESPACE)
if(HOST_ARCH MATCHES "x86_64|amd64|AMD64")
set(download_filename "ninja-linux-${program_version}.zip")
set(download_urls "https://github.com/ninja-build/ninja/releases/download/v${program_version}/ninja-linux.zip")
set(tool_subdirectory "${program_version}-linux")
set(paths_to_search "${DOWNLOADS}/tools/ninja-${program_version}-linux")
set(download_sha512 93e802e9c17fb59636cddde4bad1ddaadad624f4ecfee00d5c78790330a4e9d433180e795718cda27da57215ce643c3929cf72c85337ee019d868c56f2deeef3)
elseif(HOST_ARCH MATCHES "arm64|aarch64|ARM64|AARCH64")
# No official aarch64 build for 1.10.2, fall back to system ninja
if(DEFINED ENV{VCPKG_FORCE_SYSTEM_BINARIES})
set(program_name)
set(paths_to_search)
return()
endif()
message(STATUS "Ninja 1.10.2 not available for ARM64, will try to find system ninja")
set(program_name)
set(paths_to_search)
return()
else()
if(DEFINED ENV{VCPKG_FORCE_SYSTEM_BINARIES})
set(program_name)
set(paths_to_search)
return()
endif()
message(FATAL_ERROR "Unsupported host architecture for prebuilt ninja: ${HOST_ARCH}")
endif()
else()
message(FATAL_ERROR "Unsupported host platform: ${CMAKE_HOST_SYSTEM_NAME}")
endif()

2 changes: 1 addition & 1 deletion cpp/vcpkg
Open in desktop
Submodule vcpkg updated 2178 files
2 changes: 1 addition & 1 deletion cpp/vcpkg.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@
],
"overrides": [
{ "name": "openssl", "version-string": "3.3.0" },
{ "name": "arcticdb-sparrow", "version": "1.1.0" },
{ "name": "arcticdb-sparrow", "version": "1.3.0" },
{ "name": "arrow", "version": "18.1.0" },
{ "name": "aws-sdk-cpp", "version": "1.11.474", "$note": "Update overlay json to upgrade; Upgrade to >=1.11.486 blocked by default integrity change" },
{ "name": "aws-crt-cpp", "version": "0.29.7" },
Expand Down
2 changes: 1 addition & 1 deletion environment-dev.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ dependencies:
- openssl
- libcurl
- bitmagic
- sparrow-devel==1.2.0
- sparrow-devel==1.3.0
- spdlog
- azure-core-cpp
- azure-identity-cpp
Expand Down
3 changes: 3 additions & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ test-extras = "Testing"
before-all = """
[[ -e /etc/arcticdb_deps_installed ]] || { yum update -y && yum install -y zip \
openssl-devel cyrus-sasl-devel devtoolset-11 devtoolset-11-libatomic-devel libcurl-devel ; }
# Override vcpkg ninja to use older version (1.10.2) compatible with GLIBCXX_3.4.19
cp '{project}/cpp/third_party/vcpkg_overlays/ninja/vcpkg_find_acquire_program(NINJA).cmake' \
'{project}/cpp/vcpkg/scripts/cmake/vcpkg_find_acquire_program(NINJA).cmake'
{project}/build_tooling/prep_cpp_build.sh
{project}/build_tooling/vcpkg_caching.sh
"""
Expand Down
Loading