@@ -512,10 +512,24 @@ void J9::RecognizedCallTransformer::process_java_lang_StrictMath_and_Math_sqrt(T
512
512
/*
513
513
Transform an Unsafe atomic call to diamonds with equivalent semantics
514
514
515
+ If OffHeap is enabled, a runtime isArray check is added to determine
516
+ if loading the dataAddrPtr is necessary.
517
+
515
518
yes
516
519
isObjectNull [A] ------------------------------------------>
517
520
| |
518
521
| no |
522
+ #if OffHeap |
523
+ | yes |
524
+ isArray [I] ------------------------> |
525
+ | | |
526
+ | use the dataAddrPtr of the array |
527
+ | xcall atomic method helper [J] |
528
+ | | |
529
+ | [H] |
530
+ | no |
531
+ | |
532
+ #endif OffHeap |
519
533
| yes |
520
534
isNotLowTagged [B] ---------------------------------------->
521
535
| |
@@ -562,6 +576,18 @@ ifacmpeq -> <Block_E>
562
576
aconst null
563
577
end Block_A
564
578
579
+ #if OffHeap
580
+ start Block_I
581
+ ifacmpne -> <Block_J>
582
+ andi
583
+ lloadi <isClassDepthAndFlags>
584
+ aloadi <vft-symbol>
585
+ aload <object-1>
586
+ iconst 0x10000 // array-flag
587
+ iconst 0
588
+ end Block_I
589
+ #endif OffHeap
590
+
565
591
start Block_B
566
592
iflcmpne -> <Block_E>
567
593
land
@@ -600,6 +626,20 @@ xstore
600
626
goto --> block_H
601
627
end Block_G
602
628
629
+ #if OffHeap
630
+ start Block_J
631
+ xcall atomic method helper
632
+ aladd
633
+ aloadi <contiguousArrayDataAddrField>
634
+ aload <object-1>
635
+ lload <offset>
636
+ xload value
637
+ xstore
638
+ ==>xcall
639
+ goto --> block_H
640
+ end Block_J
641
+ #endif OffHeap
642
+
603
643
start Block_E
604
644
xcall atomic method helper
605
645
aladd
@@ -658,10 +698,52 @@ void J9::RecognizedCallTransformer::processUnsafeAtomicCall(TR::TreeTop* treetop
658
698
if (isNotStaticField)
659
699
{
660
700
// It is safe to skip diamond, the address can be calculated directly via [object+offset]
661
- address = comp ()->target ().is32Bit () ? TR::Node::create (TR::aiadd, 2 , objectNode, TR::Node::create (TR::l2i, 1 , offsetNode)) :
662
- TR::Node::create (TR::aladd, 2 , objectNode, offsetNode);
663
- if (enableTrace)
664
- traceMsg (comp (), " Field is not static, use the object and offset directly\n " );
701
+
702
+ // Except if OffHeap is used, then check if object is array
703
+ #if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
704
+ if (TR::Compiler->om .isOffHeapAllocationEnabled ())
705
+ {
706
+ // Generate null-check and array-check blocks
707
+ TR::TransformUtil::createTempsForCall (this , treetop);
708
+ objectNode = unsafeCall->getChild (1 );
709
+ offsetNode = unsafeCall->getChild (2 );
710
+
711
+ // Test if object is null
712
+ isObjectNullNode = TR::Node::createif (TR::ifacmpeq, objectNode->duplicateTree (), TR::Node::aconst (0 ), NULL );
713
+ isObjectNullTreeTop = TR::TreeTop::create (comp (), isObjectNullNode);
714
+ treetop->insertBefore (isObjectNullTreeTop);
715
+ treetop->getEnclosingBlock ()->split (treetop, cfg, fixupCommoning);
716
+
717
+ if (enableTrace)
718
+ traceMsg (comp (), " Created isObjectNull test node n%dn, non-null object will fall through to Block_%d\n " , isObjectNullNode->getGlobalIndex (), treetop->getEnclosingBlock ()->getNumber ());
719
+
720
+ // generate array check treetop
721
+ TR::Node *vftLoad = TR::Node::createWithSymRef (TR::aloadi, 1 , 1 ,
722
+ objectNode->duplicateTree (),
723
+ comp ()->getSymRefTab ()->findOrCreateVftSymbolRef ());
724
+
725
+ isObjectArrayNode = TR::Node::createif (TR::ificmpne,
726
+ comp ()->fej9 ()->testIsClassArrayType (vftLoad),
727
+ TR::Node::create (TR::iconst, 0 ),
728
+ NULL );
729
+ isObjectArrayTreeTop = TR::TreeTop::create (comp (), isObjectArrayNode, NULL , NULL );
730
+ treetop->insertBefore (isObjectArrayTreeTop);
731
+ treetop->getEnclosingBlock ()->split (treetop, cfg, fixupCommoning);
732
+
733
+ if (enableTrace)
734
+ traceMsg (comp (), " Created isObjectArray test node n%dn, array will branch to array access block\n " , isObjectArrayNode->getGlobalIndex ());
735
+
736
+ address = comp ()->target ().is32Bit () ? TR::Node::create (TR::aiadd, 2 , objectNode->duplicateTree (), TR::Node::create (TR::l2i, 1 , offsetNode->duplicateTree ())) :
737
+ TR::Node::create (TR::aladd, 2 , objectNode->duplicateTree (), offsetNode->duplicateTree ());
738
+ }
739
+ else
740
+ #endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */
741
+ {
742
+ address = comp ()->target ().is32Bit () ? TR::Node::create (TR::aiadd, 2 , objectNode, TR::Node::create (TR::l2i, 1 , offsetNode)) :
743
+ TR::Node::create (TR::aladd, 2 , objectNode, offsetNode);
744
+ if (enableTrace)
745
+ traceMsg (comp (), " Field is not static, use the object and offset directly\n " );
746
+ }
665
747
}
666
748
else
667
749
{
@@ -795,7 +877,8 @@ void J9::RecognizedCallTransformer::processUnsafeAtomicCall(TR::TreeTop* treetop
795
877
unsafeCall->removeChild (1 ); // remove object node
796
878
unsafeCall->setSymbolReference (comp ()->getSymRefTab ()->findOrCreateCodeGenInlinedHelper (helper));
797
879
798
- if (!isNotStaticField)
880
+ // Setup and connect check blocks if generated
881
+ if (isObjectNullTreeTop)
799
882
{
800
883
// Split so that the return value from the atomic method helper call will be stored into a temp if required
801
884
TR::TreeTop *nextTreeTop = treetop->getNextTreeTop ();
@@ -817,54 +900,17 @@ void J9::RecognizedCallTransformer::processUnsafeAtomicCall(TR::TreeTop* treetop
817
900
}
818
901
}
819
902
820
- /* Example of the atomic method helper
821
- * n92n treetop
822
- * n93n icall <atomicFetchAndAdd>
823
- * n94n aladd
824
- * n95n aload <temp slot 5>
825
- * n96n lload <temp slot 3>
826
- * n97n iload <temp slot 4>
827
- */
828
- // Create another helper call that loads from ramStatics
829
- TR::TreeTop *unsafeCallRamStaticsTT = treetop->duplicateTree ();
830
- TR::Node *unsafeCallRamStaticsNode = unsafeCallRamStaticsTT->getNode ()->getFirstChild ();
831
- TR::Node *addressNode = unsafeCallRamStaticsNode->getFirstChild ();
832
- TR::Node *loadNode = addressNode->getFirstChild ();
833
-
834
- loadNode->setSymbolReference (newSymbolReference); // Use the same symRef as the objectAdjustmentNode
835
- treetop->insertBefore (unsafeCallRamStaticsTT);
836
-
837
- // Store the return value from the helper call that loads from ramStatics
838
- if (storeReturnNode)
839
- {
840
- TR::Node *storeNode = TR::Node::createStore (unsafeCall, storeReturnNode->getSymbolReference (), unsafeCallRamStaticsNode);
841
- treetop->insertBefore (TR::TreeTop::create (comp (), storeNode));
842
- }
843
-
844
- // Insert goto from the helper call that loads from ramStatics to the final return block
845
- TR::Node *gotoNode = TR::Node::create (unsafeCall, TR::Goto);
846
- gotoNode->setBranchDestination (returnBlock->getEntry ());
847
- treetop->insertBefore (TR::TreeTop::create (comp (), gotoNode));
848
-
849
- if (enableTrace)
850
- traceMsg (comp (), " Created atomic method helper block_%d that loads from ramStatics treetop n%dn. returnBlock block_%d\n " ,
851
- unsafeCallRamStaticsTT->getEnclosingBlock ()->getNumber (), unsafeCallRamStaticsTT->getNode ()->getGlobalIndex (), returnBlock->getNumber ());
852
-
853
- // Split the block that contains the original helper call into a separate block
854
- treetop->getEnclosingBlock ()->split (treetop, cfg, fixupCommoning);
855
-
856
- // Create another helper call for array access (offheap only)
857
- #if defined (J9VM_GC_SPARSE_HEAP_ALLOCATION)
858
- if (isObjectArrayTreeTop != NULL )
903
+ // If isNotStaticField and array-check is generated then
904
+ // setup and connect the null and array check blocks
905
+ #if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
906
+ if (isNotStaticField && isObjectArrayTreeTop)
859
907
{
908
+ // Generate array access block
860
909
TR::TreeTop *arrayAccessTreeTop = treetop->duplicateTree ();
861
910
TR::Node *addrToAccessNode = arrayAccessTreeTop->getNode ()->getChild (0 )->getChild (0 );
862
911
TR::Block *arrayAccessBlock = TR::Block::createEmptyBlock (arrayAccessTreeTop->getNode (), comp (),
863
912
treetop->getEnclosingBlock ()->getFrequency ());
864
913
arrayAccessBlock->append (arrayAccessTreeTop);
865
- arrayAccessBlock->append (TR::TreeTop::create (comp (), TR::Node::create (arrayAccessTreeTop->getNode (),
866
- TR::Goto, 0 ,
867
- returnBlock->getEntry ())));
868
914
869
915
// load dataAddr
870
916
TR::Node *objBaseAddrNode = addrToAccessNode->getChild (0 );
@@ -877,10 +923,13 @@ void J9::RecognizedCallTransformer::processUnsafeAtomicCall(TR::TreeTop* treetop
877
923
878
924
// set as array test destination and insert array access into IL trees
879
925
// - if object is array, goto array access block
880
- // - else, fall through to lowtag test
881
- unsafeCallRamStaticsTT ->getEnclosingBlock ()->getExit ()->insertTreeTopsAfterMe (arrayAccessBlock->getEntry (), arrayAccessBlock->getExit ());
926
+ // - else, fall through to non-array access
927
+ treetop ->getEnclosingBlock ()->getExit ()->insertTreeTopsAfterMe (arrayAccessBlock->getEntry (), arrayAccessBlock->getExit ());
882
928
isObjectArrayNode->setBranchDestination (arrayAccessTreeTop->getEnclosingBlock ()->getEntry ());
883
929
930
+ treetop->getEnclosingBlock ()->append (TR::TreeTop::create (comp (), TR::Node::create (arrayAccessTreeTop->getNode (),
931
+ TR::Goto, 0 ,
932
+ returnBlock->getEntry ())));
884
933
cfg->addNode (arrayAccessBlock);
885
934
cfg->addEdge (TR::CFGEdge::createEdge (isObjectArrayTreeTop->getEnclosingBlock (), arrayAccessBlock, comp ()->trMemory ()));
886
935
cfg->addEdge (TR::CFGEdge::createEdge (arrayAccessBlock, returnBlock, comp ()->trMemory ()));
@@ -892,23 +941,111 @@ void J9::RecognizedCallTransformer::processUnsafeAtomicCall(TR::TreeTop* treetop
892
941
arrayAccessTreeTop->insertTreeTopsAfterMe (TR::TreeTop::create (comp (), storeNode));
893
942
}
894
943
944
+ isObjectNullNode->setBranchDestination (treetop->getEnclosingBlock ()->getEntry ());
945
+ cfg->addEdge (TR::CFGEdge::createEdge (isObjectNullTreeTop->getEnclosingBlock (), treetop->getEnclosingBlock (), comp ()->trMemory ()));
946
+
895
947
if (enableTrace)
896
948
traceMsg (comp (), " Created array access helper block_%d that loads dataAddr pointer from array object address\n " , arrayAccessBlock->getNumber ());
897
949
}
898
- #endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */
950
+ else
951
+ #endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */
952
+ // If not isNotStaticField then setup and connect all
953
+ // generated check blocks
954
+ if (!isNotStaticField)
955
+ {
956
+ // Create another helper call that loads from ramStatics
957
+ /* Example of the atomic method helper
958
+ * n92n treetop
959
+ * n93n icall <atomicFetchAndAdd>
960
+ * n94n aladd
961
+ * n95n aload <temp slot 5>
962
+ * n96n lload <temp slot 3>
963
+ * n97n iload <temp slot 4>
964
+ */
965
+ TR::TreeTop *unsafeCallRamStaticsTT = treetop->duplicateTree ();
966
+ TR::Node *unsafeCallRamStaticsNode = unsafeCallRamStaticsTT->getNode ()->getFirstChild ();
967
+ TR::Node *addressNode = unsafeCallRamStaticsNode->getFirstChild ();
968
+ TR::Node *loadNode = addressNode->getFirstChild ();
969
+
970
+ loadNode->setSymbolReference (newSymbolReference); // Use the same symRef as the objectAdjustmentNode
971
+ treetop->insertBefore (unsafeCallRamStaticsTT);
972
+
973
+ // Store the return value from the helper call that loads from ramStatics
974
+ if (storeReturnNode)
975
+ {
976
+ TR::Node *storeNode = TR::Node::createStore (unsafeCall, storeReturnNode->getSymbolReference (), unsafeCallRamStaticsNode);
977
+ treetop->insertBefore (TR::TreeTop::create (comp (), storeNode));
978
+ }
899
979
900
- // Setup CFG edges
901
- cfg->addEdge (unsafeCallRamStaticsTT->getEnclosingBlock (), returnBlock);
980
+ // Insert goto from the helper call that loads from ramStatics to the final return block
981
+ TR::Node *gotoNode = TR::Node::create (unsafeCall, TR::Goto);
982
+ gotoNode->setBranchDestination (returnBlock->getEntry ());
983
+ treetop->insertBefore (TR::TreeTop::create (comp (), gotoNode));
902
984
903
- if (enableTrace)
904
- traceMsg (comp (), " Block_%d contains call to atomic method helper, and is the target of isObjectNull and isNotLowTagged tests\n " , treetop->getEnclosingBlock ()->getNumber ());
985
+ if (enableTrace)
986
+ traceMsg (comp (), " Created atomic method helper block_%d that loads from ramStatics treetop n%dn. returnBlock block_%d\n " ,
987
+ unsafeCallRamStaticsTT->getEnclosingBlock ()->getNumber (), unsafeCallRamStaticsTT->getNode ()->getGlobalIndex (), returnBlock->getNumber ());
905
988
906
- isObjectNullNode->setBranchDestination (treetop->getEnclosingBlock ()->getEntry ());
907
- cfg->addEdge (TR::CFGEdge::createEdge (isObjectNullTreeTop->getEnclosingBlock (), treetop->getEnclosingBlock (), comp ()->trMemory ()));
908
- isNotLowTaggedNode->setBranchDestination (treetop->getEnclosingBlock ()->getEntry ());
909
- cfg->addEdge (TR::CFGEdge::createEdge (isNotLowTaggedTreeTop->getEnclosingBlock (), treetop->getEnclosingBlock (), comp ()->trMemory ()));
989
+ // Split the block that contains the original helper call into a separate block
990
+ treetop->getEnclosingBlock ()->split (treetop, cfg, fixupCommoning);
910
991
911
- cfg->removeEdge (unsafeCallRamStaticsTT->getEnclosingBlock (), treetop->getEnclosingBlock ());
992
+ // Create another helper call for array access (offheap only)
993
+ #if defined (J9VM_GC_SPARSE_HEAP_ALLOCATION)
994
+ if (isObjectArrayTreeTop != NULL )
995
+ {
996
+ TR::TreeTop *arrayAccessTreeTop = treetop->duplicateTree ();
997
+ TR::Node *addrToAccessNode = arrayAccessTreeTop->getNode ()->getChild (0 )->getChild (0 );
998
+ TR::Block *arrayAccessBlock = TR::Block::createEmptyBlock (arrayAccessTreeTop->getNode (), comp (),
999
+ treetop->getEnclosingBlock ()->getFrequency ());
1000
+ arrayAccessBlock->append (arrayAccessTreeTop);
1001
+ arrayAccessBlock->append (TR::TreeTop::create (comp (), TR::Node::create (arrayAccessTreeTop->getNode (),
1002
+ TR::Goto, 0 ,
1003
+ returnBlock->getEntry ())));
1004
+
1005
+ // load dataAddr
1006
+ TR::Node *objBaseAddrNode = addrToAccessNode->getChild (0 );
1007
+ TR::Node *dataAddrNode = TR::TransformUtil::generateDataAddrLoadTrees (comp (), objBaseAddrNode);
1008
+ addrToAccessNode->setChild (0 , dataAddrNode);
1009
+
1010
+ // correct refcounts
1011
+ objBaseAddrNode->decReferenceCount ();
1012
+ dataAddrNode->incReferenceCount ();
1013
+
1014
+ // set as array test destination and insert array access into IL trees
1015
+ // - if object is array, goto array access block
1016
+ // - else, fall through to lowtag test
1017
+ unsafeCallRamStaticsTT->getEnclosingBlock ()->getExit ()->insertTreeTopsAfterMe (arrayAccessBlock->getEntry (), arrayAccessBlock->getExit ());
1018
+ isObjectArrayNode->setBranchDestination (arrayAccessTreeTop->getEnclosingBlock ()->getEntry ());
1019
+
1020
+ cfg->addNode (arrayAccessBlock);
1021
+ cfg->addEdge (TR::CFGEdge::createEdge (isObjectArrayTreeTop->getEnclosingBlock (), arrayAccessBlock, comp ()->trMemory ()));
1022
+ cfg->addEdge (TR::CFGEdge::createEdge (arrayAccessBlock, returnBlock, comp ()->trMemory ()));
1023
+
1024
+ // Store the return value from the helper call for array access
1025
+ if (storeReturnNode)
1026
+ {
1027
+ TR::Node *storeNode = TR::Node::createStore (unsafeCall, storeReturnNode->getSymbolReference (), arrayAccessTreeTop->getNode ()->getFirstChild ());
1028
+ arrayAccessTreeTop->insertTreeTopsAfterMe (TR::TreeTop::create (comp (), storeNode));
1029
+ }
1030
+
1031
+ if (enableTrace)
1032
+ traceMsg (comp (), " Created array access helper block_%d that loads dataAddr pointer from array object address\n " , arrayAccessBlock->getNumber ());
1033
+ }
1034
+ #endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */
1035
+
1036
+ // Setup CFG edges
1037
+ cfg->addEdge (unsafeCallRamStaticsTT->getEnclosingBlock (), returnBlock);
1038
+
1039
+ if (enableTrace)
1040
+ traceMsg (comp (), " Block_%d contains call to atomic method helper, and is the target of isObjectNull and isNotLowTagged tests\n " , treetop->getEnclosingBlock ()->getNumber ());
1041
+
1042
+ isObjectNullNode->setBranchDestination (treetop->getEnclosingBlock ()->getEntry ());
1043
+ cfg->addEdge (TR::CFGEdge::createEdge (isObjectNullTreeTop->getEnclosingBlock (), treetop->getEnclosingBlock (), comp ()->trMemory ()));
1044
+ isNotLowTaggedNode->setBranchDestination (treetop->getEnclosingBlock ()->getEntry ());
1045
+ cfg->addEdge (TR::CFGEdge::createEdge (isNotLowTaggedTreeTop->getEnclosingBlock (), treetop->getEnclosingBlock (), comp ()->trMemory ()));
1046
+
1047
+ cfg->removeEdge (unsafeCallRamStaticsTT->getEnclosingBlock (), treetop->getEnclosingBlock ());
1048
+ }
912
1049
}
913
1050
}
914
1051
0 commit comments