Skip to content

Commit

Permalink
Add functionality for converting mlarrow record batches to pyarrow re…
Browse files Browse the repository at this point in the history
…cord batches
  • Loading branch information
sgilmore10 committed May 10, 2024
1 parent 3a477ff commit 2eff6ab
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 8 deletions.
17 changes: 13 additions & 4 deletions matlab/src/matlab/+arrow/+c/mlarrow2pyarrow.m
Original file line number Diff line number Diff line change
@@ -1,13 +1,22 @@
function pyarrowArray = mlarrow2pyarrow(mlarrowArray)
[cArrayWrapper, cSchemaWrapper] = pyrunfile(fullfile(pwd, "+internal/FFIWrapper.py"), ["cArrayWrapper" "cSchemaWrapper"]);
folder = fileparts(mfilename("fullpath"));

[cArrayWrapper, cSchemaWrapper] = pyrunfile(fullfile(folder, "+internal/FFIWrapper.py"), ["cArrayWrapper" "cSchemaWrapper"]);

cArrayAddress = uint64(cArrayWrapper.getAddress());
cSchemaAdress = uint64(cSchemaWrapper.getAddress());

mlarrowArray.exportToC(cArrayAddress, cSchemaAdress);

dummyArray = py.pyarrow.array([1 2]);
importFunc = py.getattr(dummyArray, "_import_from_c");
pyarrowArray = importFunc(cArrayAddress, cSchemaAdress);
if isa(mlarrowArray, "arrow.array.Array")
dummyArray = py.pyarrow.array([1 2]);
importFunc = py.getattr(dummyArray, "_import_from_c");
pyarrowArray = importFunc(cArrayAddress, cSchemaAdress);
else
dummyArray = py.pyarrow.array([1 2]);
dummyRB = py.pyarrow.record_batch(py.list({dummyArray}), names={'Var1'});
importFunc = py.getattr(dummyRB, "_import_from_c");
pyarrowArray = importFunc(cArrayAddress, cSchemaAdress);
end
end

8 changes: 6 additions & 2 deletions matlab/src/matlab/+arrow/+c/pyarrow2mlarrow.m
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@
exportFunc = py.getattr(pyarrowArray, "_export_to_c");
exportFunc(cArray.Address, cSchema.Address);

mlarrowArray = arrow.array.Array.importFromC(cArray, cSchema);

if isa(pyarrowArray, "py.pyarrow.lib.Array")
mlarrowArray = arrow.array.Array.importFromC(cArray, cSchema);
else
importer = arrow.c.internal.RecordBatchImporter();
mlarrowArray = importer.import(cArray, cSchema);
end
end
11 changes: 9 additions & 2 deletions matlab/src/matlab/+arrow/+tabular/RecordBatch.m
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,15 @@ function displayScalarObject(obj)
end

methods(Hidden)
function exportToC(cArrayAddress, cSchemaAddress)
obj.Proxy.exportToC(cArrayAddress, cSchemaAddress);
function exportToC(obj, cArrayAddress, cSchemaAddress)
arguments
obj(1, 1) arrow.tabular.RecordBatch
cArrayAddress(1, 1) uint64
cSchemaAddress(1, 1) uint64
end
args = struct(ArrowArrayAddress=cArrayAddress,...
ArrowSchemaAddress=cSchemaAddress);
obj.Proxy.exportToC(args);
end
end
end

0 comments on commit 2eff6ab

Please sign in to comment.