Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize array access macros to reduce using is-contiguous check #20888

Merged
merged 1 commit into from
Jan 16, 2025
Merged

Optimize array access macros to reduce using is-contiguous check #20888

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion runtime/gc_glue_java/EnvironmentDelegate.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ class MM_EnvironmentDelegate
#if defined(J9VM_ENV_DATA64)
if (objectModel->isIndexable(objectPtr)) {
GC_ArrayObjectModel *indexableObjectModel = &_extensions->indexableObjectModel;
if (_vmThread->isVirtualLargeObjectHeapEnabled && indexableObjectModel->isInlineContiguousArraylet((J9IndexableObject *)objectPtr)) {
if (indexableObjectModel->isVirtualLargeObjectHeapEnabled() && indexableObjectModel->isInlineContiguousArraylet((J9IndexableObject *)objectPtr)) {
_gcEnv._shouldFixupDataAddrForContiguous = indexableObjectModel->shouldFixupDataAddrForContiguous((J9IndexableObject *)objectPtr);
} else {
_gcEnv._shouldFixupDataAddrForContiguous = false;
Expand Down
2 changes: 1 addition & 1 deletion runtime/gc_include/ObjectAllocationAPI.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ class MM_ObjectAllocationAPI
: _gcAllocationType(currentThread->javaVM->gcAllocationType)
, _contiguousIndexableHeaderSize(currentThread->contiguousIndexableHeaderSize)
#if defined(J9VM_ENV_DATA64)
, _isIndexableDataAddrPresent(currentThread->isIndexableDataAddrPresent)
, _isIndexableDataAddrPresent(currentThread->javaVM->isIndexableDataAddrPresent)
#endif /* defined(J9VM_ENV_DATA64) */
#if defined(J9VM_GC_BATCH_CLEAR_TLH)
, _initializeSlotsOnTLHAllocate(currentThread->javaVM->initializeSlotsOnTLHAllocate)
Expand Down
2 changes: 1 addition & 1 deletion runtime/gc_modron_startup/mminit.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2972,7 +2972,7 @@ gcInitializeDefaults(J9JavaVM* vm)
#if defined(J9VM_ENV_DATA64)
vm->isIndexableDualHeaderShapeEnabled = TRUE;
vm->isIndexableDataAddrPresent = FALSE;
vm->isVirtualLargeObjectHeapEnabled = FALSE;
vm->indexableObjectLayout = J9IndexableObjectLayout_NoDataAddr_NoArraylet;
#endif /* defined(J9VM_ENV_DATA64) */

/* enable estimateFragmentation for all GCs as default for java, but not the estimated result would not affect concurrentgc kickoff by default */
Expand Down
4 changes: 3 additions & 1 deletion runtime/gc_realtime/ConfigurationRealtime.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,9 @@ MM_Heap *
MM_ConfigurationRealtime::createHeapWithManager(MM_EnvironmentBase *env, uintptr_t heapBytesRequested, MM_HeapRegionManager *regionManager)
{
MM_GCExtensionsBase *extensions = env->getExtensions();

J9JavaVM *vm = (J9JavaVM *)extensions->getOmrVM()->_language_vm;
/* Let VM know that Metronome GC has discontiguous indexable object (arraylet layout) */
vm->indexableObjectLayout = J9IndexableObjectLayout_NoDataAddr_Arraylet;
#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
PORT_ACCESS_FROM_ENVIRONMENT(env);

Expand Down
11 changes: 8 additions & 3 deletions runtime/gc_vlhgc/ConfigurationIncrementalGenerational.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,11 @@ MM_ConfigurationIncrementalGenerational::createHeapWithManager(MM_EnvironmentBas

#if defined(J9VM_ENV_DATA64)
extensions->indexableObjectModel.setIsDataAddressPresent(true);
J9JavaVM *vm = (J9JavaVM *)extensions->getOmrVM()->_language_vm;
/* Let VM know that indexable objects in Balanced always have dataAddr, and
let's assume initially it has arraylets, which can be later overridden if Offheap is Enabled.
*/
vm->indexableObjectLayout = J9IndexableObjectLayout_DataAddr_Arraylet;
#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
if (extensions->isVirtualLargeObjectHeapRequested) {
/* Create off-heap */
Expand All @@ -144,9 +149,9 @@ MM_ConfigurationIncrementalGenerational::createHeapWithManager(MM_EnvironmentBas
extensions->largeObjectVirtualMemory = largeObjectVirtualMemory;
extensions->indexableObjectModel.setEnableVirtualLargeObjectHeap(true);
extensions->isVirtualLargeObjectHeapEnabled = true;
/* reset vm->isVirtualLargeObjectHeapEnabled and vm->unsafeIndexableHeaderSize for off-heap case */
J9JavaVM *vm = (J9JavaVM *)extensions->getOmrVM()->_language_vm;
vm->isVirtualLargeObjectHeapEnabled = TRUE;
/* Overriding the original assumption that Balanced has arraylets. */
vm->indexableObjectLayout = J9IndexableObjectLayout_DataAddr_NoArraylet;
/* reset vm->unsafeIndexableHeaderSize for off-heap case */
vm->unsafeIndexableHeaderSize = 0;
} else {
#if defined(OMR_GC_VLHGC_CONCURRENT_COPY_FORWARD)
Expand Down
2 changes: 1 addition & 1 deletion runtime/gc_vlhgc/VLHGCAccessBarrier.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ MM_VLHGCAccessBarrier::indexableDataDisplacement(J9VMThread *vmThread, J9Indexab
IDATA displacement = 0;

#if defined(J9VM_ENV_DATA64)
Assert_MM_true(vmThread->isVirtualLargeObjectHeapEnabled);
Assert_MM_true(_extensions->isVirtualLargeObjectHeapEnabled);
/* Adjacency check against dst object since src object may be overwritten during sliding compaction. */
if (_extensions->indexableObjectModel.isDataAdjacentToHeader(dst))
#endif /* defined(J9VM_ENV_DATA64) */
Expand Down
78 changes: 41 additions & 37 deletions runtime/oti/j9accessbarrier.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,17 +123,17 @@ typedef struct J9IndexableObject* mm_j9array_t;
? (&((elemType*)((((J9IndexableObjectContiguousCompressed *)(array)) + 1)))[index]) \
: (&((elemType*)((((J9IndexableObjectContiguousFull *)(array)) + 1)))[index]))

#if defined(J9VM_ENV_DATA64)
#define J9JAVAARRAYCONTIGUOUS_WITH_DATAADDRESS_VIRTUALLARGEOBJECTHEAPDISABLED_EA(vmThread, array, index, elemType) \
#define J9ISCONTIGUOUSARRAY(vmThread, array) \
(J9VMTHREAD_COMPRESS_OBJECT_REFERENCES(vmThread) \
? (&((elemType*)((((J9IndexableObjectWithDataAddressContiguousCompressed *)(array)) + 1)))[index]) \
: (&((elemType*)((((J9IndexableObjectWithDataAddressContiguousFull *)(array)) + 1)))[index]))
? (0 != ((J9IndexableObjectContiguousCompressed *)(array))->size) \
: (0 != ((J9IndexableObjectContiguousFull *)(array))->size))

#define J9JAVAARRAYCONTIGUOUS_WITH_DATAADDRESS_VIRTUALLARGEOBJECTHEAPDISABLED_EA_VM(javaVM, array, index, elemType) \
#define J9ISCONTIGUOUSARRAY_VM(javaVM, array) \
(J9JAVAVM_COMPRESS_OBJECT_REFERENCES(javaVM) \
? (&((elemType*)((((J9IndexableObjectWithDataAddressContiguousCompressed *)(array)) + 1)))[index]) \
: (&((elemType*)((((J9IndexableObjectWithDataAddressContiguousFull *)(array)) + 1)))[index]))
? (0 != ((J9IndexableObjectContiguousCompressed *)(array))->size) \
: (0 != ((J9IndexableObjectContiguousFull *)(array))->size))

#if defined(J9VM_ENV_DATA64)
#define J9JAVAARRAYCONTIGUOUS_WITH_DATAADDRESS_VIRTUALLARGEOBJECTHEAPENABLED_EA(vmThread, array, index, elemType) \
(J9VMTHREAD_COMPRESS_OBJECT_REFERENCES(vmThread) \
? (&((elemType*)((((J9IndexableObjectWithDataAddressContiguousCompressed *)(array))->dataAddr)))[index]) \
Expand All @@ -145,41 +145,45 @@ typedef struct J9IndexableObject* mm_j9array_t;
: (&((elemType*)((((J9IndexableObjectWithDataAddressContiguousFull *)(array))->dataAddr)))[index]))

#define J9JAVAARRAYCONTIGUOUS_EA(vmThread, array, index, elemType) \
(((vmThread)->isIndexableDataAddrPresent) \
? (((vmThread)->isVirtualLargeObjectHeapEnabled) \
? J9JAVAARRAYCONTIGUOUS_WITH_DATAADDRESS_VIRTUALLARGEOBJECTHEAPENABLED_EA(vmThread, array, index, elemType) \
: J9JAVAARRAYCONTIGUOUS_WITH_DATAADDRESS_VIRTUALLARGEOBJECTHEAPDISABLED_EA(vmThread, array, index, elemType)) \
: J9JAVAARRAYCONTIGUOUS_BASE_EA(vmThread, array, index, elemType))
Copy link
Contributor

@dmitripivkine dmitripivkine Jan 14, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we are removing only usage of two DISABLED macros above. These macros can be removed.

(&((elemType*)((((UDATA)(array)) + (vmThread)->contiguousIndexableHeaderSize)))[(index)])

#define J9JAVAARRAYCONTIGUOUS_EA_VM(javaVM, array, index, elemType) \
(((javaVM)->isIndexableDataAddrPresent) \
? (((javaVM)->isVirtualLargeObjectHeapEnabled) \
? J9JAVAARRAYCONTIGUOUS_WITH_DATAADDRESS_VIRTUALLARGEOBJECTHEAPENABLED_EA_VM(javaVM, array, index, elemType) \
: J9JAVAARRAYCONTIGUOUS_WITH_DATAADDRESS_VIRTUALLARGEOBJECTHEAPDISABLED_EA_VM(javaVM, array, index, elemType)) \
: J9JAVAARRAYCONTIGUOUS_BASE_EA_VM(javaVM, array, index, elemType))

#else /* defined(J9VM_ENV_DATA64) */
#define J9JAVAARRAYCONTIGUOUS_EA(vmThread, array, index, elemType) \
J9JAVAARRAYCONTIGUOUS_BASE_EA(vmThread, array, index, elemType)

#define J9JAVAARRAYCONTIGUOUS_EA_VM(javaVM, array, index, elemType) \
J9JAVAARRAYCONTIGUOUS_BASE_EA_VM(javaVM, array, index, elemType)

#endif /* defined(J9VM_ENV_DATA64) */
(&((elemType*)((((UDATA)(array)) + (javaVM)->contiguousIndexableHeaderSize)))[(index)])

#define J9ISCONTIGUOUSARRAY(vmThread, array) \
(J9VMTHREAD_COMPRESS_OBJECT_REFERENCES(vmThread) \
? (0 != ((J9IndexableObjectContiguousCompressed *)(array))->size) \
: (0 != ((J9IndexableObjectContiguousFull *)(array))->size))

#define J9ISCONTIGUOUSARRAY_VM(javaVM, array) \
(J9JAVAVM_COMPRESS_OBJECT_REFERENCES(javaVM) \
? (0 != ((J9IndexableObjectContiguousCompressed *)(array))->size) \
: (0 != ((J9IndexableObjectContiguousFull *)(array))->size))
/*
* if standard GC (J9IndexableObjectLayout_NoDataAddr_NoArraylet)
* contiguous-base
* else if off-heap enabled (J9IndexableObjectLayout_DataAddr_NoArraylet)
* contiguous-via-dataAddr
* else balancedGC with off-heap disabled or Metronome GC (J9IndexableObjectLayout_NoDataAddr_Arraylet or J9IndexableObjectLayout_DataAddr_Arraylet)
* if contigious
* contiguous
* else
* discontigous
*/
#define J9JAVAARRAY_EA(vmThread, array, index, elemType) \
((J9IndexableObjectLayout_NoDataAddr_NoArraylet == (vmThread)->indexableObjectLayout) \
? J9JAVAARRAYCONTIGUOUS_BASE_EA(vmThread, array, index, elemType) \
: ((J9IndexableObjectLayout_DataAddr_NoArraylet == (vmThread)->indexableObjectLayout) \
? J9JAVAARRAYCONTIGUOUS_WITH_DATAADDRESS_VIRTUALLARGEOBJECTHEAPENABLED_EA(vmThread, array, index, elemType) \
: (J9ISCONTIGUOUSARRAY(vmThread, array) \
? J9JAVAARRAYCONTIGUOUS_EA(vmThread, array, index, elemType) \
: J9JAVAARRAYDISCONTIGUOUS_EA(vmThread, array, index, elemType))))

#define J9JAVAARRAY_EA_VM(javaVM, array, index, elemType) \
((J9IndexableObjectLayout_NoDataAddr_NoArraylet == (javaVM)->indexableObjectLayout) \
? J9JAVAARRAYCONTIGUOUS_BASE_EA_VM(javaVM, array, index, elemType) \
: ((J9IndexableObjectLayout_DataAddr_NoArraylet == (javaVM)->indexableObjectLayout) \
? J9JAVAARRAYCONTIGUOUS_WITH_DATAADDRESS_VIRTUALLARGEOBJECTHEAPENABLED_EA_VM(javaVM, array, index, elemType) \
: (J9ISCONTIGUOUSARRAY_VM(javaVM, array) \
? J9JAVAARRAYCONTIGUOUS_EA_VM(javaVM, array, index, elemType) \
: J9JAVAARRAYDISCONTIGUOUS_EA_VM(javaVM, array, index, elemType))))

#else /* defined(J9VM_ENV_DATA64) */
/* TODO: queries compressed twice - optimize? */
#define J9JAVAARRAY_EA(vmThread, array, index, elemType) (J9ISCONTIGUOUSARRAY(vmThread, array) ? J9JAVAARRAYCONTIGUOUS_EA(vmThread, array, index, elemType) : J9JAVAARRAYDISCONTIGUOUS_EA(vmThread, array, index, elemType))
#define J9JAVAARRAY_EA_VM(javaVM, array, index, elemType) (J9ISCONTIGUOUSARRAY_VM(javaVM, array) ? J9JAVAARRAYCONTIGUOUS_EA_VM(javaVM, array, index, elemType) : J9JAVAARRAYDISCONTIGUOUS_EA_VM(javaVM, array, index, elemType))
#define J9JAVAARRAY_EA(vmThread, array, index, elemType) (J9ISCONTIGUOUSARRAY(vmThread, array) ? J9JAVAARRAYCONTIGUOUS_BASE_EA(vmThread, array, index, elemType) : J9JAVAARRAYDISCONTIGUOUS_EA(vmThread, array, index, elemType))
#define J9JAVAARRAY_EA_VM(javaVM, array, index, elemType) (J9ISCONTIGUOUSARRAY_VM(javaVM, array) ? J9JAVAARRAYCONTIGUOUS_BASE_EA_VM(javaVM, array, index, elemType) : J9JAVAARRAYDISCONTIGUOUS_EA_VM(javaVM, array, index, elemType))
#endif /* defined(J9VM_ENV_DATA64) */

/*
* Private helpers for reference field types
Expand Down
20 changes: 17 additions & 3 deletions runtime/oti/j9nonbuilder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,13 @@
#define J9ArrayShape32Bit 0x2
#define J9ArrayShape64Bit 0x3

/* J9IndexableObjectLayout_xxx are used in indexableObjectLayout field in J9VMThread and J9JavaVM */
#define J9IndexableObjectLayout_NoDataAddr_NoArraylet 0x0 /* StandardGC case */
#define J9IndexableObjectLayout_NoDataAddr_Arraylet 0x1 /* Metronome GC case */
#define J9IndexableObjectLayout_DataAddr_NoArraylet 0x2 /* Balanced GC Offheap enabled case */
#define J9IndexableObjectLayout_DataAddr_Arraylet 0x3 /* Balanced GC Offheap disabled case */
#define J9IndexableObjectLayout_DataAddrMask 0x2
#define J9IndexableObjectLayout_ArrayletMask 0x1
/* @ddr_namespace: map_to_type=J9ClassInitFlags */

/* Constants from J9ClassInitFlags */
Expand Down Expand Up @@ -5540,8 +5547,7 @@ typedef struct J9VMThread {
UDATA contiguousIndexableHeaderSize;
UDATA discontiguousIndexableHeaderSize;
#if defined(J9VM_ENV_DATA64)
UDATA isIndexableDataAddrPresent;
U_32 isVirtualLargeObjectHeapEnabled;
U_32 indexableObjectLayout;
#endif /* defined(J9VM_ENV_DATA64) */
void* gpInfo;
void* jitVMwithThreadInfo;
Expand Down Expand Up @@ -5660,6 +5666,10 @@ typedef struct J9VMThread {
#endif /* defined(J9VM_OPT_JFR) */
} J9VMThread;

#if defined(J9VM_ENV_DATA64)
#define J9VMTHREAD_IS_ARRAYLET_ENABLED(vmThread) (J9IndexableObjectLayout_ArrayletMask & (vmThread)->indexableObjectLayout)
#define J9VMTHREAD_IS_INDEXBLE_DATAADDR_PRESENT(vmThread) (J9IndexableObjectLayout_DataAddrMask & (vmThread)->indexableObjectLayout)
#endif
#define J9VMTHREAD_ALIGNMENT 0x100
#define J9VMTHREAD_RESERVED_C_STACK_FRACTION 8
#define J9VMTHREAD_STATE_RUNNING 1
Expand Down Expand Up @@ -6106,7 +6116,7 @@ typedef struct J9JavaVM {
UDATA discontiguousIndexableHeaderSize;
#if defined(J9VM_ENV_DATA64)
UDATA isIndexableDataAddrPresent;
U_32 isVirtualLargeObjectHeapEnabled;
U_32 indexableObjectLayout;
U_32 isIndexableDualHeaderShapeEnabled;
#endif /* defined(J9VM_ENV_DATA64) */
struct J9VMThread* exclusiveVMAccessQueueHead;
Expand Down Expand Up @@ -6347,6 +6357,10 @@ typedef struct J9JavaVM {
#define J9JAVAVM_OBJECT_HEADER_SIZE(vm) (J9JAVAVM_COMPRESS_OBJECT_REFERENCES(vm) ? sizeof(J9ObjectCompressed) : sizeof(J9ObjectFull))
#define J9JAVAVM_CONTIGUOUS_INDEXABLE_HEADER_SIZE(vm) ((vm)->contiguousIndexableHeaderSize)
#define J9JAVAVM_DISCONTIGUOUS_INDEXABLE_HEADER_SIZE(vm) ((vm)->discontiguousIndexableHeaderSize)
#if defined(J9VM_ENV_DATA64)
#define J9JAVAVM_IS_ARRAYLET_ENABLED(vm) (J9IndexableObjectLayout_ArrayletMask & (vm)->indexableObjectLayout)
#define J9JAVAVM_IS_INDEXBLE_DATAADDR_PRESENT(vm) (J9IndexableObjectLayout_DataAddrMask & (vm)->indexableObjectLayout)eObjectLayout)
#endif

#if JAVA_SPEC_VERSION >= 16
/* The mask for the signature type identifier */
Expand Down
3 changes: 1 addition & 2 deletions runtime/vm/vmthread.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,8 +196,7 @@ allocateVMThread(J9JavaVM *vm, omrthread_t osThread, UDATA privateFlags, void *m
newThread->discontiguousIndexableHeaderSize = vm->discontiguousIndexableHeaderSize;
newThread->unsafeIndexableHeaderSize = vm->unsafeIndexableHeaderSize;
#if defined(J9VM_ENV_DATA64)
newThread->isIndexableDataAddrPresent = vm->isIndexableDataAddrPresent;
newThread->isVirtualLargeObjectHeapEnabled = vm->isVirtualLargeObjectHeapEnabled;
newThread->indexableObjectLayout = vm->indexableObjectLayout;
#endif /* defined(J9VM_ENV_DATA64) */

newThread->privateFlags = privateFlags;
Expand Down