Skip to content

Commit e54b7df

Browse files
authored
feat: add Decimal32/Decimal64 support (#683)
Initial implementation of Decimal32/Decimal64 support in nanoarrow.
1 parent 253b7ec commit e54b7df

16 files changed

+482
-43
lines changed

.github/workflows/build-and-test-device.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ jobs:
8585
if: steps.cache-arrow-build.outputs.cache-hit != 'true'
8686
shell: bash
8787
run: |
88-
ci/scripts/build-arrow-cpp-minimal.sh 15.0.2 arrow
88+
ci/scripts/build-arrow-cpp-minimal.sh 18.0.0 arrow
8989
9090
- name: Build
9191
run: |

.github/workflows/build-and-test-ipc.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ jobs:
7878
if: steps.cache-arrow-build.outputs.cache-hit != 'true'
7979
shell: bash
8080
run: |
81-
ci/scripts/build-arrow-cpp-minimal.sh 15.0.2 arrow
81+
ci/scripts/build-arrow-cpp-minimal.sh 18.0.0 arrow
8282
8383
- name: Build
8484
run: |

.github/workflows/build-and-test.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ jobs:
7070
if: steps.cache-arrow-build.outputs.cache-hit != 'true'
7171
shell: bash
7272
run: |
73-
ci/scripts/build-arrow-cpp-minimal.sh 15.0.2 arrow
73+
ci/scripts/build-arrow-cpp-minimal.sh 18.0.0 arrow
7474
7575
- name: Build nanoarrow
7676
run: |
@@ -154,7 +154,7 @@ jobs:
154154
if: steps.cache-arrow-build.outputs.cache-hit != 'true'
155155
shell: bash
156156
run: |
157-
ci/scripts/build-arrow-cpp-minimal.sh 16.0.0 arrow
157+
ci/scripts/build-arrow-cpp-minimal.sh 18.0.0 arrow
158158
159159
- name: Run meson testing script
160160
run: |

.github/workflows/clang-tidy.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ jobs:
5454
if: steps.cache-arrow-build.outputs.cache-hit != 'true'
5555
shell: bash
5656
run: |
57-
ci/scripts/build-arrow-cpp-minimal.sh 15.0.2 arrow
57+
ci/scripts/build-arrow-cpp-minimal.sh 18.0.0 arrow
5858
5959
- name: Build nanoarrow
6060
run: |

ci/docker/alpine.dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ RUN apk add bash linux-headers git cmake R R-dev g++ gfortran gnupg curl py3-vir
2323

2424
# For Arrow C++
2525
COPY ci/scripts/build-arrow-cpp-minimal.sh /
26-
RUN /build-arrow-cpp-minimal.sh 15.0.2 /arrow
26+
RUN /build-arrow-cpp-minimal.sh 18.0.0 /arrow
2727

2828
# There's a missing define that numpy's build needs on s390x and there is no wheel
2929
RUN (grep -e "S390" /usr/include/bits/hwcap.h && echo "#define HWCAP_S390_VX HWCAP_S390_VXRS" >> /usr/include/bits/hwcap.h) || true

src/nanoarrow/common/array.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,8 @@ static ArrowErrorCode ArrowArraySetStorageType(struct ArrowArray* array,
104104
case NANOARROW_TYPE_HALF_FLOAT:
105105
case NANOARROW_TYPE_FLOAT:
106106
case NANOARROW_TYPE_DOUBLE:
107+
case NANOARROW_TYPE_DECIMAL32:
108+
case NANOARROW_TYPE_DECIMAL64:
107109
case NANOARROW_TYPE_DECIMAL128:
108110
case NANOARROW_TYPE_DECIMAL256:
109111
case NANOARROW_TYPE_INTERVAL_MONTHS:

src/nanoarrow/common/array_test.cc

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1245,6 +1245,94 @@ TEST(ArrayTest, ArrayTestAppendToIntervalArrayMonthDayNano) {
12451245
#endif
12461246
}
12471247

1248+
TEST(ArrayTest, ArrayTestAppendToDecimal32Array) {
1249+
struct ArrowArray array;
1250+
struct ArrowDecimal decimal;
1251+
1252+
ArrowDecimalInit(&decimal, 32, 8, 3);
1253+
ASSERT_EQ(ArrowArrayInitFromType(&array, NANOARROW_TYPE_DECIMAL32), NANOARROW_OK);
1254+
EXPECT_EQ(ArrowArrayStartAppending(&array), NANOARROW_OK);
1255+
1256+
ArrowDecimalSetInt(&decimal, 12345);
1257+
EXPECT_EQ(ArrowArrayAppendDecimal(&array, &decimal), NANOARROW_OK);
1258+
1259+
EXPECT_EQ(ArrowArrayAppendNull(&array, 2), NANOARROW_OK);
1260+
1261+
ArrowDecimalSetInt(&decimal, -67890);
1262+
EXPECT_EQ(ArrowArrayAppendDecimal(&array, &decimal), NANOARROW_OK);
1263+
1264+
EXPECT_EQ(ArrowArrayFinishBuildingDefault(&array, nullptr), NANOARROW_OK);
1265+
EXPECT_EQ(array.length, 4);
1266+
EXPECT_EQ(array.null_count, 2);
1267+
auto validity_buffer = reinterpret_cast<const uint8_t*>(array.buffers[0]);
1268+
auto data_buffer = reinterpret_cast<const uint8_t*>(array.buffers[1]);
1269+
EXPECT_EQ(validity_buffer[0], 0b00001001);
1270+
1271+
ArrowDecimalSetInt(&decimal, 12345);
1272+
EXPECT_EQ(memcmp(data_buffer, decimal.words, 4), 0);
1273+
ArrowDecimalSetInt(&decimal, -67890);
1274+
EXPECT_EQ(memcmp(data_buffer + 3 * 4, decimal.words, 4), 0);
1275+
1276+
#if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) && ARROW_VERSION_MAJOR >= 18
1277+
auto arrow_array = ImportArray(&array, decimal32(8, 3));
1278+
ARROW_EXPECT_OK(arrow_array);
1279+
1280+
auto builder = Decimal32Builder(decimal32(8, 3));
1281+
ARROW_EXPECT_OK(builder.Append(*Decimal32::FromString("12.345")));
1282+
ARROW_EXPECT_OK(builder.AppendNulls(2));
1283+
ARROW_EXPECT_OK(builder.Append(*Decimal32::FromString("-67.890")));
1284+
auto expected_array = builder.Finish();
1285+
1286+
EXPECT_TRUE(arrow_array.ValueUnsafe()->Equals(expected_array.ValueUnsafe()));
1287+
#else
1288+
ArrowArrayRelease(&array);
1289+
#endif
1290+
}
1291+
1292+
TEST(ArrayTest, ArrayTestAppendToDecimal64Array) {
1293+
struct ArrowArray array;
1294+
struct ArrowDecimal decimal;
1295+
1296+
ArrowDecimalInit(&decimal, 64, 10, 3);
1297+
ASSERT_EQ(ArrowArrayInitFromType(&array, NANOARROW_TYPE_DECIMAL64), NANOARROW_OK);
1298+
EXPECT_EQ(ArrowArrayStartAppending(&array), NANOARROW_OK);
1299+
1300+
ArrowDecimalSetInt(&decimal, 12345);
1301+
EXPECT_EQ(ArrowArrayAppendDecimal(&array, &decimal), NANOARROW_OK);
1302+
1303+
EXPECT_EQ(ArrowArrayAppendNull(&array, 2), NANOARROW_OK);
1304+
1305+
ArrowDecimalSetInt(&decimal, -67890);
1306+
EXPECT_EQ(ArrowArrayAppendDecimal(&array, &decimal), NANOARROW_OK);
1307+
1308+
EXPECT_EQ(ArrowArrayFinishBuildingDefault(&array, nullptr), NANOARROW_OK);
1309+
EXPECT_EQ(array.length, 4);
1310+
EXPECT_EQ(array.null_count, 2);
1311+
auto validity_buffer = reinterpret_cast<const uint8_t*>(array.buffers[0]);
1312+
auto data_buffer = reinterpret_cast<const uint8_t*>(array.buffers[1]);
1313+
EXPECT_EQ(validity_buffer[0], 0b00001001);
1314+
1315+
ArrowDecimalSetInt(&decimal, 12345);
1316+
EXPECT_EQ(memcmp(data_buffer, decimal.words, 8), 0);
1317+
ArrowDecimalSetInt(&decimal, -67890);
1318+
EXPECT_EQ(memcmp(data_buffer + 3 * 8, decimal.words, 8), 0);
1319+
1320+
#if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) && ARROW_VERSION_MAJOR >= 18
1321+
auto arrow_array = ImportArray(&array, decimal64(10, 3));
1322+
ARROW_EXPECT_OK(arrow_array);
1323+
1324+
auto builder = Decimal64Builder(decimal64(10, 3));
1325+
ARROW_EXPECT_OK(builder.Append(*Decimal64::FromString("12.345")));
1326+
ARROW_EXPECT_OK(builder.AppendNulls(2));
1327+
ARROW_EXPECT_OK(builder.Append(*Decimal64::FromString("-67.890")));
1328+
auto expected_array = builder.Finish();
1329+
1330+
EXPECT_TRUE(arrow_array.ValueUnsafe()->Equals(expected_array.ValueUnsafe()));
1331+
#else
1332+
ArrowArrayRelease(&array);
1333+
#endif
1334+
}
1335+
12481336
TEST(ArrayTest, ArrayTestAppendToDecimal128Array) {
12491337
struct ArrowArray array;
12501338
struct ArrowDecimal decimal;
@@ -3821,6 +3909,82 @@ TEST(ArrayViewTest, ArrayViewTestGetIntervalMonthDayNano) {
38213909
ArrowArrayRelease(&array);
38223910
}
38233911

3912+
#if ARROW_VERSION_MAJOR >= 18
3913+
TEST(ArrayViewTest, ArrayViewTestGetDecimal32) {
3914+
struct ArrowArray array;
3915+
struct ArrowSchema schema;
3916+
struct ArrowArrayView array_view;
3917+
struct ArrowError error;
3918+
3919+
auto type = decimal32(8, 3);
3920+
3921+
// Array with nulls
3922+
auto builder = Decimal32Builder(type);
3923+
ARROW_EXPECT_OK(builder.Append(*Decimal32::FromReal(1.234, 8, 3)));
3924+
ARROW_EXPECT_OK(builder.AppendNulls(2));
3925+
ARROW_EXPECT_OK(builder.Append(*Decimal32::FromReal(-5.678, 8, 3)));
3926+
auto maybe_arrow_array = builder.Finish();
3927+
ARROW_EXPECT_OK(maybe_arrow_array);
3928+
auto arrow_array = maybe_arrow_array.ValueUnsafe();
3929+
3930+
ARROW_EXPECT_OK(ExportArray(*arrow_array, &array, &schema));
3931+
ASSERT_EQ(ArrowArrayViewInitFromSchema(&array_view, &schema, &error), NANOARROW_OK);
3932+
ASSERT_EQ(ArrowArrayViewSetArray(&array_view, &array, &error), NANOARROW_OK);
3933+
EXPECT_EQ(ArrowArrayViewValidate(&array_view, NANOARROW_VALIDATION_LEVEL_FULL, &error),
3934+
NANOARROW_OK);
3935+
3936+
ArrowDecimal decimal;
3937+
ArrowDecimalInit(&decimal, 32, 8, 3);
3938+
3939+
ArrowArrayViewGetDecimalUnsafe(&array_view, 0, &decimal);
3940+
EXPECT_EQ(ArrowDecimalGetIntUnsafe(&decimal), 1234);
3941+
3942+
ArrowArrayViewGetDecimalUnsafe(&array_view, 3, &decimal);
3943+
EXPECT_EQ(ArrowDecimalGetIntUnsafe(&decimal), -5678);
3944+
3945+
ArrowArrayViewReset(&array_view);
3946+
ArrowSchemaRelease(&schema);
3947+
ArrowArrayRelease(&array);
3948+
}
3949+
3950+
TEST(ArrayViewTest, ArrayViewTestGetDecimal64) {
3951+
struct ArrowArray array;
3952+
struct ArrowSchema schema;
3953+
struct ArrowArrayView array_view;
3954+
struct ArrowError error;
3955+
3956+
auto type = decimal64(10, 3);
3957+
3958+
// Array with nulls
3959+
auto builder = Decimal64Builder(type);
3960+
ARROW_EXPECT_OK(builder.Append(*Decimal64::FromReal(1.234, 10, 3)));
3961+
ARROW_EXPECT_OK(builder.AppendNulls(2));
3962+
ARROW_EXPECT_OK(builder.Append(*Decimal64::FromReal(-5.678, 10, 3)));
3963+
auto maybe_arrow_array = builder.Finish();
3964+
ARROW_EXPECT_OK(maybe_arrow_array);
3965+
auto arrow_array = maybe_arrow_array.ValueUnsafe();
3966+
3967+
ARROW_EXPECT_OK(ExportArray(*arrow_array, &array, &schema));
3968+
ASSERT_EQ(ArrowArrayViewInitFromSchema(&array_view, &schema, &error), NANOARROW_OK);
3969+
ASSERT_EQ(ArrowArrayViewSetArray(&array_view, &array, &error), NANOARROW_OK);
3970+
EXPECT_EQ(ArrowArrayViewValidate(&array_view, NANOARROW_VALIDATION_LEVEL_FULL, &error),
3971+
NANOARROW_OK);
3972+
3973+
ArrowDecimal decimal;
3974+
ArrowDecimalInit(&decimal, 64, 10, 3);
3975+
3976+
ArrowArrayViewGetDecimalUnsafe(&array_view, 0, &decimal);
3977+
EXPECT_EQ(ArrowDecimalGetIntUnsafe(&decimal), 1234);
3978+
3979+
ArrowArrayViewGetDecimalUnsafe(&array_view, 3, &decimal);
3980+
EXPECT_EQ(ArrowDecimalGetIntUnsafe(&decimal), -5678);
3981+
3982+
ArrowArrayViewReset(&array_view);
3983+
ArrowSchemaRelease(&schema);
3984+
ArrowArrayRelease(&array);
3985+
}
3986+
#endif
3987+
38243988
TEST(ArrayViewTest, ArrayViewTestGetDecimal128) {
38253989
struct ArrowArray array;
38263990
struct ArrowSchema schema;

src/nanoarrow/common/inline_array.h

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -700,6 +700,22 @@ static inline ArrowErrorCode ArrowArrayAppendDecimal(struct ArrowArray* array,
700700
struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1);
701701

702702
switch (private_data->storage_type) {
703+
case NANOARROW_TYPE_DECIMAL32:
704+
if (value->n_words != 0) {
705+
return EINVAL;
706+
} else {
707+
NANOARROW_RETURN_NOT_OK(
708+
ArrowBufferAppend(data_buffer, value->words, sizeof(uint32_t)));
709+
break;
710+
}
711+
case NANOARROW_TYPE_DECIMAL64:
712+
if (value->n_words != 1) {
713+
return EINVAL;
714+
} else {
715+
NANOARROW_RETURN_NOT_OK(
716+
ArrowBufferAppend(data_buffer, value->words, sizeof(uint64_t)));
717+
break;
718+
}
703719
case NANOARROW_TYPE_DECIMAL128:
704720
if (value->n_words != 2) {
705721
return EINVAL;
@@ -1267,6 +1283,12 @@ static inline void ArrowArrayViewGetDecimalUnsafe(const struct ArrowArrayView* a
12671283
i += array_view->offset;
12681284
const uint8_t* data_view = array_view->buffer_views[1].data.as_uint8;
12691285
switch (array_view->storage_type) {
1286+
case NANOARROW_TYPE_DECIMAL32:
1287+
ArrowDecimalSetBytes(out, data_view + (i * 4));
1288+
break;
1289+
case NANOARROW_TYPE_DECIMAL64:
1290+
ArrowDecimalSetBytes(out, data_view + (i * 8));
1291+
break;
12701292
case NANOARROW_TYPE_DECIMAL128:
12711293
ArrowDecimalSetBytes(out, data_view + (i * 16));
12721294
break;

0 commit comments

Comments
 (0)