Skip to content

Commit 54ae00e

Browse files
jirkamarsikjirkamarsik
committed
[GR-59082] [GR-59083] Grow memories faster.
PullRequest: graal/19874
2 parents 0a48e5c + f76ce56 commit 54ae00e

File tree

6 files changed

+101
-36
lines changed

6 files changed

+101
-36
lines changed

‎wasm/src/org.graalvm.wasm.test/src/org/graalvm/wasm/test/WasmJsApiSuite.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import static org.graalvm.wasm.utils.WasmBinaryTools.compileWat;
4444

4545
import java.io.IOException;
46+
import java.nio.ByteBuffer;
4647
import java.nio.ByteOrder;
4748
import java.util.Arrays;
4849
import java.util.HashMap;
@@ -2388,6 +2389,21 @@ public void testReimportWasmFunctionViaImportObject() throws IOException, Interr
23882389
}
23892390
}
23902391

2392+
@Test
2393+
public void testSharedMemoryGrow() throws IOException {
2394+
runTest(options -> options.option("wasm.UseUnsafeMemory", "true"), context -> {
2395+
WasmMemory sharedMemory = WebAssembly.memAlloc(1, 2, true);
2396+
ByteBuffer preGrowBuffer = WebAssembly.memAsByteBuffer(sharedMemory);
2397+
long previousSize = WebAssembly.memGrow(sharedMemory, 1);
2398+
Assert.assertEquals("Wrong previous size reported by mem.grow", 1, previousSize);
2399+
ByteBuffer postGrowBuffer = WebAssembly.memAsByteBuffer(sharedMemory);
2400+
preGrowBuffer.put(0, (byte) 42);
2401+
Assert.assertEquals("Value written to pre-grow buffer not seen in post-grow buffer", 42, postGrowBuffer.get(0));
2402+
postGrowBuffer.put(1, (byte) 21);
2403+
Assert.assertEquals("Value written to post-grow buffer not seen in pre-grow buffer", 21, preGrowBuffer.get(1));
2404+
});
2405+
}
2406+
23912407
private static void runMemoryTest(Consumer<WasmContext> testCase) throws IOException {
23922408
runTest(null, testCase);
23932409
runTest(options -> options.option("wasm.UseUnsafeMemory", "true"), testCase);

‎wasm/src/org.graalvm.wasm/src/org/graalvm/wasm/api/WebAssembly.java

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -866,16 +866,19 @@ public static int invokeMemWaitCallback(Node node, WasmMemory memory, long addre
866866

867867
private static Object memAsByteBuffer(Object[] args) {
868868
checkArgumentCount(args, 1);
869-
if (args[0] instanceof WasmMemory) {
870-
WasmMemory memory = (WasmMemory) args[0];
871-
ByteBuffer buffer = WasmMemoryLibrary.getUncached().asByteBuffer(memory);
869+
if (args[0] instanceof WasmMemory memory) {
870+
ByteBuffer buffer = memAsByteBuffer(memory);
872871
if (buffer != null) {
873872
return WasmContext.get(null).environment().asGuestValue(buffer);
874873
}
875874
}
876875
return WasmConstant.VOID;
877876
}
878877

878+
public static ByteBuffer memAsByteBuffer(WasmMemory memory) {
879+
return WasmMemoryLibrary.getUncached().asByteBuffer(memory);
880+
}
881+
879882
private Object globalAlloc(Object[] args) {
880883
checkArgumentCount(args, 2);
881884
InteropLibrary lib = InteropLibrary.getUncached();

‎wasm/src/org.graalvm.wasm/src/org/graalvm/wasm/memory/ByteArrayWasmMemory.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,14 +68,14 @@ final class ByteArrayWasmMemory extends WasmMemory {
6868
public static final long MAX_ALLOWED_SIZE = Integer.MAX_VALUE / MEMORY_PAGE_SIZE;
6969

7070
@TruffleBoundary
71-
private ByteArrayWasmMemory(long declaredMinSize, long declaredMaxSize, long initialSize, long maxAllowedSize, boolean indexType64, boolean shared) {
72-
super(declaredMinSize, declaredMaxSize, initialSize, maxAllowedSize, indexType64, shared);
71+
private ByteArrayWasmMemory(long declaredMinSize, long declaredMaxSize, long initialSize, long maxAllowedSize, boolean indexType64) {
72+
super(declaredMinSize, declaredMaxSize, initialSize, maxAllowedSize, indexType64, false);
7373
this.dynamicBuffer = allocateStatic(initialSize * MEMORY_PAGE_SIZE);
7474
}
7575

7676
@TruffleBoundary
77-
ByteArrayWasmMemory(long declaredMinSize, long declaredMaxSize, boolean indexType64, boolean shared) {
78-
this(declaredMinSize, declaredMaxSize, declaredMinSize, Math.min(declaredMaxSize, MAX_ALLOWED_SIZE), indexType64, shared);
77+
ByteArrayWasmMemory(long declaredMinSize, long declaredMaxSize, boolean indexType64) {
78+
this(declaredMinSize, declaredMaxSize, declaredMinSize, Math.min(declaredMaxSize, MAX_ALLOWED_SIZE), indexType64);
7979
}
8080

8181
private byte[] buffer() {
@@ -1049,7 +1049,7 @@ public void copyFrom(WasmMemory source, long sourceOffset, long destinationOffse
10491049

10501050
@ExportMessage
10511051
public WasmMemory duplicate() {
1052-
final ByteArrayWasmMemory other = new ByteArrayWasmMemory(declaredMinSize, declaredMaxSize, size(), maxAllowedSize, indexType64, shared);
1052+
final ByteArrayWasmMemory other = new ByteArrayWasmMemory(declaredMinSize, declaredMaxSize, size(), maxAllowedSize, indexType64);
10531053
System.arraycopy(buffer(), 0, other.buffer(), 0, (int) byteSize());
10541054
return other;
10551055
}

‎wasm/src/org.graalvm.wasm/src/org/graalvm/wasm/memory/NativeWasmMemory.java

Lines changed: 48 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -67,30 +67,38 @@
6767
final class NativeWasmMemory extends WasmMemory {
6868

6969
private long startAddress;
70+
/**
71+
* The visible size of the Wasm linear memory.
72+
*/
7073
private long size;
74+
/**
75+
* The actual size of the memory buffer allocated by GraalWasm.
76+
*/
77+
private long bufferSize;
7178

7279
public static final long MAX_ALLOWED_SIZE = Sizes.MAX_MEMORY_64_INSTANCE_SIZE;
7380

7481
private static final Unsafe unsafe;
7582
private static final VarHandle SIZE_FIELD;
7683

7784
@TruffleBoundary
78-
private NativeWasmMemory(long declaredMinSize, long declaredMaxSize, long initialSize, long maxAllowedSize, boolean indexType64, boolean shared) {
79-
super(declaredMinSize, declaredMaxSize, initialSize, maxAllowedSize, indexType64, shared);
85+
private NativeWasmMemory(long declaredMinSize, long declaredMaxSize, long initialSize, long maxAllowedSize, boolean indexType64) {
86+
super(declaredMinSize, declaredMaxSize, initialSize, maxAllowedSize, indexType64, false);
8087
this.size = declaredMinSize;
81-
final long byteSize = byteSize();
82-
this.startAddress = allocate(byteSize);
88+
final long initialBufferSize = byteSize();
89+
this.startAddress = allocate(initialBufferSize);
90+
this.bufferSize = initialBufferSize;
8391
}
8492

8593
@TruffleBoundary
86-
NativeWasmMemory(long declaredMinSize, long declaredMaxSize, boolean indexType64, boolean shared) {
87-
this(declaredMinSize, declaredMaxSize, declaredMinSize, WasmMath.minUnsigned(declaredMaxSize, MAX_ALLOWED_SIZE), indexType64, shared);
94+
NativeWasmMemory(long declaredMinSize, long declaredMaxSize, boolean indexType64) {
95+
this(declaredMinSize, declaredMaxSize, declaredMinSize, WasmMath.minUnsigned(declaredMaxSize, MAX_ALLOWED_SIZE), indexType64);
8896
}
8997

90-
private static long allocate(long byteSize) {
98+
private static long allocate(long newBufferSize) {
9199
try {
92-
final long address = unsafe.allocateMemory(byteSize);
93-
unsafe.setMemory(address, byteSize, (byte) 0);
100+
final long address = unsafe.allocateMemory(newBufferSize);
101+
unsafe.setMemory(address, newBufferSize, (byte) 0);
94102
return address;
95103
} catch (OutOfMemoryError error) {
96104
throw WasmException.create(Failure.MEMORY_ALLOCATION_FAILED);
@@ -118,13 +126,26 @@ public synchronized long grow(long extraPageSize) {
118126
// Condition above and limit on maxAllowedSize (see NativeWasmMemory#MAX_ALLOWED_SIZE)
119127
// ensure computation of targetByteSize does not overflow.
120128
final long targetByteSize = Math.multiplyExact(Math.addExact(previousSize, extraPageSize), MEMORY_PAGE_SIZE);
121-
final long updatedSize = previousSize + extraPageSize;
122-
try {
123-
startAddress = unsafe.reallocateMemory(startAddress, targetByteSize);
124-
unsafe.setMemory(startAddress + byteSize(), targetByteSize - byteSize(), (byte) 0);
125-
} catch (OutOfMemoryError error) {
126-
throw WasmException.create(Failure.MEMORY_ALLOCATION_FAILED);
129+
if (Long.compareUnsigned(targetByteSize, bufferSize) > 0) {
130+
try {
131+
long newBufferSize = newBufferSize(targetByteSize);
132+
startAddress = unsafe.reallocateMemory(startAddress, newBufferSize);
133+
unsafe.setMemory(startAddress + bufferSize, newBufferSize - bufferSize, (byte) 0);
134+
bufferSize = newBufferSize;
135+
} catch (OutOfMemoryError error) {
136+
// Over-allocating failed, so try to allocate at least the amount of memory that
137+
// was requested.
138+
try {
139+
long newBufferSize = targetByteSize;
140+
startAddress = unsafe.reallocateMemory(startAddress, newBufferSize);
141+
unsafe.setMemory(startAddress + bufferSize, newBufferSize - bufferSize, (byte) 0);
142+
bufferSize = newBufferSize;
143+
} catch (OutOfMemoryError errorAgain) {
144+
throw WasmException.create(Failure.MEMORY_ALLOCATION_FAILED);
145+
}
146+
}
127147
}
148+
final long updatedSize = previousSize + extraPageSize;
128149
currentMinSize = updatedSize;
129150
SIZE_FIELD.setVolatile(this, updatedSize);
130151
invokeGrowCallback();
@@ -134,12 +155,21 @@ public synchronized long grow(long extraPageSize) {
134155
}
135156
}
136157

158+
public long newBufferSize(long targetByteSize) {
159+
// bufferSize <= Sizes.MAX_MEMORY_64_INSTANCE_BYTE_SIZE, so this should not overflow
160+
long prefBufferSize = Math.addExact(bufferSize, bufferSize >> 1);
161+
// maxAllowedByteSize <= Sizes.MAX_MEMORY_64_INSTANCE_BYTE_SIZE, so no overflow
162+
long maxAllowedByteSize = Math.multiplyExact(maxAllowedSize(), MEMORY_PAGE_SIZE);
163+
return Math.max(targetByteSize, Math.min(prefBufferSize, maxAllowedByteSize));
164+
}
165+
137166
@ExportMessage
138167
@TruffleBoundary
139168
public void reset() {
140169
free();
141170
size = declaredMinSize;
142-
startAddress = allocate(byteSize());
171+
bufferSize = byteSize();
172+
startAddress = allocate(bufferSize);
143173
currentMinSize = declaredMinSize;
144174
}
145175

@@ -930,7 +960,7 @@ public int atomic_wait64(Node node, long address, long expected, long timeout) {
930960

931961
@ExportMessage
932962
public WasmMemory duplicate() {
933-
final NativeWasmMemory other = new NativeWasmMemory(declaredMinSize, declaredMaxSize, size, maxAllowedSize, indexType64, shared);
963+
final NativeWasmMemory other = new NativeWasmMemory(declaredMinSize, declaredMaxSize, size, maxAllowedSize, indexType64);
934964
unsafe.copyMemory(this.startAddress, other.startAddress, this.byteSize());
935965
return other;
936966
}
@@ -967,6 +997,7 @@ public boolean freed() {
967997
@TruffleBoundary
968998
private void free() {
969999
unsafe.freeMemory(startAddress);
1000+
bufferSize = 0;
9701001
startAddress = 0;
9711002
size = 0;
9721003
}

‎wasm/src/org.graalvm.wasm/src/org/graalvm/wasm/memory/UnsafeWasmMemory.java

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,11 @@ public final class UnsafeWasmMemory extends WasmMemory {
8484
private UnsafeWasmMemory(long declaredMinSize, long declaredMaxSize, long initialSize, long maxAllowedSize, boolean indexType64, boolean shared) {
8585
super(declaredMinSize, declaredMaxSize, initialSize, maxAllowedSize, indexType64, shared);
8686
this.size = declaredMinSize;
87-
final long byteSize = byteSize();
88-
this.buffer = allocateBuffer(byteSize);
87+
final long initialByteSize = byteSize();
88+
final long maxAllowedByteSize = maxAllowedSize * MEMORY_PAGE_SIZE;
89+
// For shared memories, we have to allocate the maximum allowed size in advance to avoid
90+
// having to move the memory by reallocating.
91+
this.buffer = allocateBuffer(shared ? maxAllowedByteSize : initialByteSize);
8992
this.startAddress = getBufferAddress(buffer);
9093
}
9194

@@ -129,7 +132,9 @@ private static void validateAtomicAddress(Node node, long address, int length) {
129132
@TruffleBoundary
130133
public void reset() {
131134
size = declaredMinSize;
132-
buffer = allocateBuffer(byteSize());
135+
// For shared memories, we have to allocate the maximum allowed size in advance to avoid
136+
// having to move the memory by reallocating.
137+
buffer = allocateBuffer(shared ? maxAllowedSize * MEMORY_PAGE_SIZE : declaredMinSize * MEMORY_PAGE_SIZE);
133138
startAddress = getBufferAddress(buffer);
134139
currentMinSize = declaredMinSize;
135140
}
@@ -155,13 +160,15 @@ public synchronized long grow(long extraPageSize) {
155160
// Condition above and limit on maxAllowedSize (see UnsafeWasmMemory#MAX_ALLOWED_SIZE)
156161
// ensure computation of targetByteSize does not overflow.
157162
final long targetByteSize = multiplyExact(addExact(previousSize, extraPageSize), MEMORY_PAGE_SIZE);
158-
final long sourceByteSize = byteSize();
159-
ByteBuffer updatedBuffer = allocateBuffer(targetByteSize);
160-
final long updatedStartAddress = getBufferAddress(updatedBuffer);
163+
if (compareUnsigned(targetByteSize, buffer.capacity()) > 0) {
164+
final long sourceByteSize = byteSize();
165+
ByteBuffer updatedBuffer = allocateBuffer(newBufferSize(targetByteSize));
166+
final long updatedStartAddress = getBufferAddress(updatedBuffer);
167+
unsafe.copyMemory(startAddress, updatedStartAddress, sourceByteSize);
168+
buffer = updatedBuffer;
169+
startAddress = updatedStartAddress;
170+
}
161171
final long updatedSize = previousSize + extraPageSize;
162-
unsafe.copyMemory(startAddress, updatedStartAddress, sourceByteSize);
163-
buffer = updatedBuffer;
164-
startAddress = updatedStartAddress;
165172
currentMinSize = updatedSize;
166173
SIZE_FIELD.setVolatile(this, updatedSize);
167174
invokeGrowCallback();
@@ -171,6 +178,14 @@ public synchronized long grow(long extraPageSize) {
171178
}
172179
}
173180

181+
public long newBufferSize(long targetByteSize) {
182+
// buffer.capacity() <= Integer.MAX_VALUE, so this cannot overflow
183+
long prefBufferSize = addExact(buffer.capacity(), buffer.capacity() >> 1);
184+
// maxAllowedByteSize <= Integer.MAX_VALUE, so no overflow
185+
long maxAllowedByteSize = multiplyExact(maxAllowedSize(), MEMORY_PAGE_SIZE);
186+
return Math.max(targetByteSize, Math.min(prefBufferSize, maxAllowedByteSize));
187+
}
188+
174189
// Checkstyle: stop
175190
@ExportMessage
176191
public int load_i32(Node node, long address) {

‎wasm/src/org.graalvm.wasm/src/org/graalvm/wasm/memory/WasmMemoryFactory.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,10 @@ public static WasmMemory createMemory(long declaredMinSize, long declaredMaxSize
5757
if (directByteBufferMemoryAccess || shared) {
5858
return new UnsafeWasmMemory(declaredMinSize, declaredMaxSize, indexType64, shared);
5959
} else if (declaredMaxSize > ByteArrayWasmMemory.MAX_ALLOWED_SIZE) {
60-
return new NativeWasmMemory(declaredMinSize, declaredMaxSize, indexType64, shared);
60+
return new NativeWasmMemory(declaredMinSize, declaredMaxSize, indexType64);
6161
}
6262
}
63-
return new ByteArrayWasmMemory(declaredMinSize, declaredMaxSize, indexType64, shared);
63+
return new ByteArrayWasmMemory(declaredMinSize, declaredMaxSize, indexType64);
6464
}
6565

6666
public static long getMaximumAllowedSize(boolean shared, boolean unsafeMemory, boolean directByteBufferMemoryAccess) {

0 commit comments

Comments
 (0)