diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp index 34b5ac5b6b372..d7786cdf3e021 100644 --- a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp @@ -222,7 +222,20 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, __ pop_call_clobbered_registers(); __ bind(done); +} + +static void generate_post_barrier_same_region_check(MacroAssembler* masm, const Register store_addr, const Register new_val, const Register tmp1, Label& done) { + __ block_comment("cross-region"); + + // Does store cross heap regions? + __ eor(tmp1, store_addr, new_val); // tmp1 := store address ^ new value + __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes) + __ cbz(tmp1, done); +} +static void generate_post_barrier_null_new_value_check(MacroAssembler* masm, const Register new_val, const Register tmp1, Label& done) { + __ block_comment("null-new-val"); + __ cbz(new_val, done); } static void generate_post_barrier_fast_path(MacroAssembler* masm, @@ -232,19 +245,35 @@ static void generate_post_barrier_fast_path(MacroAssembler* masm, const Register tmp1, const Register tmp2, Label& done, - bool new_val_maybe_null) { + uint ext_barrier_data) { assert(thread == rthread, "must be"); assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg); - // Does store cross heap regions? - __ eor(tmp1, store_addr, new_val); // tmp1 := store address ^ new value - __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes) - __ cbz(tmp1, done); - // Crosses regions, storing null? - if (new_val_maybe_null) { - __ cbz(new_val, done); + bool gen_cross_region_check = ((ext_barrier_data & G1C2BarrierPostGenCrossCheck) != 0) || !UseNewCode; + bool gen_null_new_val_check = ((ext_barrier_data & G1C2BarrierPostGenNullCheck) != 0) || !UseNewCode; + bool gen_card_table_check = ((ext_barrier_data & G1C2BarrierPostGenCardCheck) != 0) || !UseNewCode; + bool null_check_first = ((ext_barrier_data & G1C2BarrierPostNullCheckFirst) != 0) || !UseNewCode; + + bool new_val_maybe_null = ((ext_barrier_data & G1C2BarrierPostNotNull) != 0); + + __ block_comment(err_msg("barrier parts: gen_same_region %d gen_null_new %d gen_card_table %d maybe_null %d swap_same_null %d", gen_cross_region_check, gen_null_new_val_check, gen_card_table_check, new_val_maybe_null, null_check_first)); + + if (!null_check_first) { + if (gen_cross_region_check) { + generate_post_barrier_same_region_check(masm, store_addr, new_val, tmp1, done); + } + // Crosses regions, storing null? + if (gen_null_new_val_check && new_val_maybe_null) { + generate_post_barrier_null_new_value_check(masm, new_val, tmp1, done); + } + } else { + assert(gen_cross_region_check, "must be"); + assert(gen_null_new_val_check, "must be"); + generate_post_barrier_null_new_value_check(masm, new_val, tmp1, done); + generate_post_barrier_same_region_check(masm, store_addr, new_val, tmp1, done); } + // Storing region crossing non-null, is card young? __ lsr(tmp1, store_addr, CardTable::card_shift()); // tmp1 := card address relative to card table base @@ -252,7 +281,7 @@ static void generate_post_barrier_fast_path(MacroAssembler* masm, Address card_table_addr(thread, in_bytes(G1ThreadLocalData::card_table_base_offset())); __ ldr(tmp2, card_table_addr); // tmp2 := card table base address __ add(tmp1, tmp1, tmp2); // tmp1 := card address - if (UseCondCardMark) { + if (gen_card_table_check && UseCondCardMark) { __ ldrb(tmp2, Address(tmp1)); // tmp2 := card // Instead of loading clean_card_val and comparing, we exploit the fact that // the LSB of non-clean cards is always 0, and the LSB of clean cards 1. @@ -262,6 +291,14 @@ static void generate_post_barrier_fast_path(MacroAssembler* masm, __ strb(zr, Address(tmp1)); // *(card address) := dirty_card_val } +static uint8_t gen_all_barrier_parts() { + return G1C2BarrierPostNotNull | G1C2BarrierPostGenCrossCheck | G1C2BarrierPostGenNullCheck | G1C2BarrierPostGenCardCheck; +} + +static uint8_t gen_no_barrier_parts() { + return 0; +} + void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, Register store_addr, Register new_val, @@ -269,7 +306,7 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, Register tmp1, Register tmp2) { Label done; - generate_post_barrier_fast_path(masm, store_addr, new_val, thread, tmp1, tmp2, done, false /* new_val_maybe_null */); + generate_post_barrier_fast_path(masm, store_addr, new_val, thread, tmp1, tmp2, done, gen_all_barrier_parts()); __ bind(done); } @@ -329,9 +366,9 @@ void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm, Register thread, Register tmp1, Register tmp2, - bool new_val_maybe_null) { + uint8_t ext_barrier_data) { Label done; - generate_post_barrier_fast_path(masm, store_addr, new_val, thread, tmp1, tmp2, done, new_val_maybe_null); + generate_post_barrier_fast_path(masm, store_addr, new_val, thread, tmp1, tmp2, done, ext_barrier_data); __ bind(done); } @@ -430,6 +467,63 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier #undef __ +#define __ masm-> + +void G1BarrierSetAssembler::g1_write_barrier_post_profile_c1(ciMethodData* md, + int bci, + MacroAssembler* masm, + Register store_addr, + Register new_val, + Register thread, + Register tmp1, + Register tmp2) { + assert(md != nullptr, "must be"); + + ciProfileData* data = md->bci_to_data(bci); + assert(data != nullptr, "must be"); + if (!data->is_G1CounterData()) { + assert(!UseNewCode, "must be"); + return; + } + + + data = data->as_G1CounterData(); + assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, rscratch1 /* FIXME: move rscratch1 to regular tmp register */); + + Register mdp = tmp2; + __ mov_metadata(mdp, md->constant_encoding()); + __ increment(Address(mdp, md->byte_offset_of_slot(data, G1CounterData::visits_counter_offset()))); + + if (UseCompressedOops) { + __ decode_heap_oop(tmp1, new_val); + __ xorptr(tmp1, tmp1, store_addr); + } else { + __ xorptr(tmp1, new_val, store_addr); + } + __ shrptr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); + __ cmp(tmp1, zr); + __ cset(tmp1, Assembler::EQ); + __ addptr(Address(mdp, md->byte_offset_of_slot(data, G1CounterData::same_region_counter_offset())), tmp1); // How many same-region pointers + + __ cmp(new_val, zr); + __ cset(tmp1, Assembler::EQ); + __ addptr(Address(mdp, md->byte_offset_of_slot(data, G1CounterData::null_new_val_counter_offset())), tmp1); // How many zeros + + __ movptr(rscratch1, Address(thread, in_bytes(G1ThreadLocalData::card_table_base_offset()))); + __ shrptr(tmp1, store_addr, G1CardTable::card_shift()); + __ cmpb(Address(tmp1, rscratch1), G1CardTable::clean_card_val()); + __ cset(tmp1, Assembler::EQ); + __ addptr(Address(mdp, md->byte_offset_of_slot(data, G1CounterData::clean_cards_counter_offset())), tmp1); // How many clean cards + + if (XXXDoYoungPreDirty) { + __ movptr(rscratch1, Address(thread, in_bytes(G1ThreadLocalData::card_table_base_offset()))); + __ shrptr(tmp1, store_addr, G1CardTable::card_shift()); + __ cmpb(Address(tmp1, rscratch1), G1CardTable::g1_young_card); + __ cset(tmp1, Assembler::EQ); + __ addptr(Address(mdp, md->byte_offset_of_slot(data, G1CounterData::from_young_counter_offset())), tmp1); // How many from-young cards + } +} + void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm, Register store_addr, Register new_val, @@ -437,10 +531,12 @@ void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm, Register tmp1, Register tmp2) { Label done; - generate_post_barrier_fast_path(masm, store_addr, new_val, thread, tmp1, tmp2, done, true /* new_val_maybe_null */); - masm->bind(done); + generate_post_barrier_fast_path(masm, store_addr, new_val, thread, tmp1, tmp2, done, gen_all_barrier_parts()); + __ bind(done); } +#undef __ + #define __ sasm-> void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.hpp index 5719fbdcc61af..5be05cc1674de 100644 --- a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.hpp @@ -90,7 +90,7 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler { Register thread, Register tmp1, Register tmp2, - bool new_val_maybe_null); + uint ext_barrier_data); #endif void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.ad b/src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.ad index 17bee0145e7c9..bd34adee5d850 100644 --- a/src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.ad +++ b/src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.ad @@ -68,7 +68,7 @@ static void write_barrier_post(MacroAssembler* masm, Assembler::InlineSkippedInstructionsCounter skip_counter(masm); G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler()); bool new_val_maybe_null = G1BarrierStubC2::post_new_val_maybe_null(node); - g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, rthread, tmp1, tmp2, new_val_maybe_null); + g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, rthread, tmp1, tmp2, G1BarrierStubC2::ext_barrier_data(node)); } %} diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp index 836caa86cb0af..7bed44bc2d8e8 100644 --- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp @@ -243,13 +243,14 @@ void InterpreterMacroAssembler::load_resolved_klass_at_offset( // Kills: // r2, r5 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass, - Label& ok_is_subtype) { + Label& ok_is_subtype, + bool is_aastore) { assert(Rsub_klass != r0, "r0 holds superklass"); assert(Rsub_klass != r2, "r2 holds 2ndary super array length"); assert(Rsub_klass != r5, "r5 holds 2ndary super array scan ptr"); // Profile the not-null value's klass. - profile_typecheck(r2, Rsub_klass, r5); // blows r2, reloads r5 + profile_typecheck(r2, Rsub_klass, r5, is_aastore); // blows r2, reloads r5 // Do the check. check_klass_subtype(Rsub_klass, r0, r2, ok_is_subtype); // blows r2 @@ -1305,19 +1306,27 @@ void InterpreterMacroAssembler::profile_ret(Register return_bci, } } -void InterpreterMacroAssembler::profile_null_seen(Register mdp) { +void InterpreterMacroAssembler::profile_null_seen(Register mdp, bool is_aastore) { if (ProfileInterpreter) { Label profile_continue; // If no method data exists, go to profile_continue. test_method_data_pointer(mdp, profile_continue); + if (UseNewCode && is_aastore) { + addptr(mdp, in_bytes(CombinedData::receiver_type_data_offset())); + } + set_mdp_flag_at(mdp, BitData::null_seen_byte_constant()); // The method data pointer needs to be updated. int mdp_delta = in_bytes(BitData::bit_data_size()); if (TypeProfileCasts) { - mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); + if (UseNewCode && is_aastore) { + mdp_delta = in_bytes(CombinedData::receiver_type_data_size()); + } else { + mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); + } } update_mdp_by_constant(mdp, mdp_delta); @@ -1325,22 +1334,35 @@ void InterpreterMacroAssembler::profile_null_seen(Register mdp) { } } -void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass, Register reg2) { +void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass, Register reg2, bool is_aastore) { if (ProfileInterpreter) { Label profile_continue; // If no method data exists, go to profile_continue. test_method_data_pointer(mdp, profile_continue); - // The method data pointer needs to be updated. - int mdp_delta = in_bytes(BitData::bit_data_size()); - if (TypeProfileCasts) { - mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); +// do no profiling for aastore here + if (TypeProfileCasts && UseNewCode && is_aastore) { + block_comment("aastore-do profiling {"); + + update_mdp_by_constant(mdp, in_bytes(CombinedData::receiver_type_data_offset())); - // Record the object type. record_klass_in_profile(klass, mdp, reg2); + + update_mdp_by_constant(mdp, in_bytes(CombinedData::receiver_type_data_size())); + + block_comment("aastore-do profiling }"); + } else { + // The method data pointer needs to be updated. + int mdp_delta = in_bytes(BitData::bit_data_size()); + if (TypeProfileCasts) { + mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); + + // Record the object type. + record_klass_in_profile(klass, mdp, reg2); + } + update_mdp_by_constant(mdp, mdp_delta); } - update_mdp_by_constant(mdp, mdp_delta); bind(profile_continue); } @@ -1396,6 +1418,99 @@ void InterpreterMacroAssembler::profile_switch_case(Register index, } } +void InterpreterMacroAssembler::profile_oop_store(Register addr_base, Register addr_index, Register new_val) { + if (!UseG1GC) { + return; + } + if (!ProfileInterpreter) { + return; + } + + Label profile_continue; + + assert_different_registers(addr_base, addr_index, new_val, rscratch1, rscratch2, r10, r11); + + block_comment("profile_oop_store {"); + + push(addr_base); + push(addr_index); + push(new_val); + push(r10); + push(r11); + + Register mdp = r10; + Register tmp = rscratch2; + Register tmp2 = r11; + // If no method data exists, exit. + test_method_data_pointer(mdp, profile_continue); + + if (UseCompressedOops) { + decode_heap_oop_not_null(new_val); + } + + xorptr(tmp, addr, new_val); + shrptr(tmp, tmp, G1HeapRegion::LogOfHRGrainBytes); + cmp(tmp, zr); + cset(tmp, Assembler::equal); + addptr(Address(mdp, in_bytes(G1CounterData::same_region_counter_offset())), tmp); + + cmp(new_val, zr); + cset(tmp, Assembler::equal); + addptr(Address(mdp, in_bytes(G1CounterData::null_new_val_counter_offset())), tmp); + + movptr(tmp, Address(thread, G1ThreadLocalData::card_table_base_offset())); + add(addr_base, addr_base, addr_index); + shrptr(addr_base, G1CardTable::card_shift()); + + cmpb(Address(tmp, addr), G1CardTable::clean_card_val()); + cset(tmp2, Assembler::notEqual); + addptr(Address(mdp, in_bytes(G1CounterData::clean_cards_counter_offset())), tmp2); + + if (XXXDoYoungPreDirty) { + cmpb(Address(tmp, addr), G1CardTable::g1_young_card); + cset(tmp2, Assembler::notEqual); + addptr(Address(mdp, in_bytes(G1CounterData::from_young_counter_offset())), tmp2); + } + + bind(profile_continue); + + pop(r11); + pop(r10); + pop(new_val); + pop(addr_index); + pop(addr_base); + + block_comment("}"); +} + +void InterpreterMacroAssembler::profile_putfield_fix_mdp() { + if (!UseG1GC) { + return; + } + if (!ProfileInterpreter) { + return; + } + + Label profile_continue; + + Register mdp = r10; + + block_comment("profile_putfield_fix_mdp {"); + + push(mdp); // Just in case. + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + addptr(Address(mdp, in_bytes(G1CounterData::visits_counter_offset())), 1); + update_mdp_by_constant(mdp, in_bytes(G1CounterData::counter_data_size())); + + bind(profile_continue); + pop(mdp); + + block_comment("}"); +} + void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) { if (state == atos) { MacroAssembler::_verify_oop_checked(reg, "broken oop", file, line); diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp index bc2070d283d7d..8cd82f122cfc6 100644 --- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp @@ -188,7 +188,7 @@ class InterpreterMacroAssembler: public MacroAssembler { // Generate a subtype check: branch to ok_is_subtype if sub_klass is // a subtype of super_klass. - void gen_subtype_check( Register sub_klass, Label &ok_is_subtype ); + void gen_subtype_check( Register sub_klass, Label &ok_is_subtype, bool is_aastore = false); // Dispatching void dispatch_prolog(TosState state, int step = 0); @@ -287,13 +287,16 @@ class InterpreterMacroAssembler: public MacroAssembler { Register scratch2, bool receiver_can_be_null = false); void profile_ret(Register return_bci, Register mdp); - void profile_null_seen(Register mdp); - void profile_typecheck(Register mdp, Register klass, Register scratch); + void profile_null_seen(Register mdp, bool is_aastore = false); + void profile_typecheck(Register mdp, Register klass, Register scratch, bool is_aastore = false); void profile_typecheck_failed(Register mdp); void profile_switch_default(Register mdp); void profile_switch_case(Register index_in_scratch, Register mdp, Register scratch2); + void profile_oop_store(Register addr_base, Register addr_index, Register new_val); + void profile_putfield_fix_mdp(); + void profile_obj_type(Register obj, const Address& mdo_addr); void profile_arguments_type(Register mdp, Register callee, Register tmp, bool is_virtual); void profile_return_type(Register mdp, Register ret, Register tmp); diff --git a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp index 60d4c3c511009..f2e49265288e6 100644 --- a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp @@ -1133,7 +1133,7 @@ void TemplateTable::aastore() { // Generate subtype check. Blows r2, r5 // Superklass in r0. Subklass in r1. - __ gen_subtype_check(r1, ok_is_subtype); + __ gen_subtype_check(r1, ok_is_subtype, true /* is_aastore */); // Come here on failure // object is at TOS @@ -1150,7 +1150,7 @@ void TemplateTable::aastore() { // Have a null in r0, r3=array, r2=index. Store null at ary[idx] __ bind(is_null); - __ profile_null_seen(r2); + __ profile_null_seen(r2, true /* is_aastore */); // Store a null do_oop_store(_masm, element_address, noreg, IS_ARRAY); @@ -2878,6 +2878,7 @@ void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteContr __ pop(atos); if (!is_static) pop_and_check_object(obj); // Store into the field + __ profile_oop_store(field.base(), field.index(), r0); do_oop_store(_masm, field, r0, IN_HEAP); if (rc == may_rewrite) { patch_bytecode(Bytecodes::_fast_aputfield, bc, r1, true, byte_no); @@ -2991,6 +2992,8 @@ void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteContr __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore); __ bind(notVolatile); } + + __ profile_putfield_fix_mdp(); } void TemplateTable::putfield(int byte_no) @@ -3094,6 +3097,7 @@ void TemplateTable::fast_storefield(TosState state) // access field switch (bytecode()) { case Bytecodes::_fast_aputfield: + __ profile_oop_store(field.base(), field.index(), r0); do_oop_store(_masm, field, r0, IN_HEAP); break; case Bytecodes::_fast_lputfield: @@ -3130,6 +3134,8 @@ void TemplateTable::fast_storefield(TosState state) __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore); __ bind(notVolatile); } + + __ profile_putfield_fix_mdp(); } diff --git a/src/hotspot/cpu/arm/templateTable_arm.cpp b/src/hotspot/cpu/arm/templateTable_arm.cpp index 0974ff1f9a9c3..3118b8f188998 100644 --- a/src/hotspot/cpu/arm/templateTable_arm.cpp +++ b/src/hotspot/cpu/arm/templateTable_arm.cpp @@ -1242,7 +1242,7 @@ void TemplateTable::aastore() { __ load_klass(Rtemp, Rarray_3); __ ldr(Rsuper_LR, Address(Rtemp, ObjArrayKlass::element_klass_offset())); - __ gen_subtype_check(Rsub_5, Rsuper_LR, throw_array_store, R0_tmp, R3_tmp); + __ gen_subtype_check(Rsub_5, Rsuper_LR, throw_array_store, R0_tmp, R3_tmp, true /* is_aastore */ ); // Come here on success // Store value diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index ff6d18e48e1a8..77bc477c85842 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -1628,6 +1628,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) { void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data, Register recv, Label* update_done) { + assert(!data->is_CombinedData(), "must be 7"); for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { Label next_test; // See if the receiver is receiver[n]. @@ -1676,6 +1677,10 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L data = md->bci_to_data(bci); assert(data != nullptr, "need data for type check"); assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); + // Forward to receiver type data. + data = data->as_ReceiverTypeData(); + //assert(!data->is_CombinedData(), "must be2"); + } Label* success_target = success; Label* failure_target = failure; @@ -1831,6 +1836,8 @@ void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { data = md->bci_to_data(bci); assert(data != nullptr, "need data for type check"); assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); + //assert(!data->is_CombinedData(), "must be"); + data = data->as_ReceiverTypeData(); } Label done; Label* success_target = &done; diff --git a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp index 4d24972bd8872..0dc301d7623e8 100644 --- a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp @@ -24,6 +24,7 @@ #include "precompiled.hpp" #include "asm/macroAssembler.inline.hpp" +#include "ci/ciMethodData.hpp" #include "gc/g1/g1BarrierSet.hpp" #include "gc/g1/g1BarrierSetAssembler.hpp" #include "gc/g1/g1BarrierSetRuntime.hpp" @@ -258,6 +259,7 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, #ifdef _LP64 assert(thread == r15_thread, "must be"); #endif // _LP64 + if (XXXSkipPreBarrier) return; Label done; Label runtime; @@ -314,6 +316,22 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, __ bind(done); } +static void generate_post_barrier_same_region_check(MacroAssembler* masm, const Register store_addr, const Register new_val, const Register tmp1, Label& done) { + __ block_comment("cross-region"); + + // Does store cross heap regions? + __ movptr(tmp1, store_addr); // tmp1 := store address + __ xorptr(tmp1, new_val); // tmp1 := store address ^ new value + __ shrptr(tmp1, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0? + __ jcc(Assembler::equal, done); +} + +static void generate_post_barrier_null_new_value_check(MacroAssembler* masm, const Register new_val, const Register tmp1, Label& done) { + __ block_comment("null-new-val"); + __ cmpptr(new_val, NULL_WORD); // new value == null? + __ jcc(Assembler::equal, done); +} + static void generate_post_barrier_fast_path(MacroAssembler* masm, const Register store_addr, const Register new_val, @@ -321,22 +339,34 @@ static void generate_post_barrier_fast_path(MacroAssembler* masm, const Register tmp1, const Register tmp2, Label& done, - bool new_val_maybe_null) { + uint8_t ext_barrier_data) { #ifdef _LP64 assert(thread == r15_thread, "must be"); #endif // _LP64 assert_different_registers(store_addr, new_val, thread, tmp1 /*, tmp2 unused */, noreg); - // Does store cross heap regions? - __ movptr(tmp1, store_addr); // tmp1 := store address - __ xorptr(tmp1, new_val); // tmp1 := store address ^ new value - __ shrptr(tmp1, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0? - __ jcc(Assembler::equal, done); + bool gen_cross_region_check = ((ext_barrier_data & G1C2BarrierPostGenCrossCheck) != 0) || !UseNewCode; + bool gen_null_new_val_check = ((ext_barrier_data & G1C2BarrierPostGenNullCheck) != 0) || !UseNewCode; + bool gen_card_table_check = ((ext_barrier_data & G1C2BarrierPostGenCardCheck) != 0) || !UseNewCode; + bool null_check_first = ((ext_barrier_data & G1C2BarrierPostNullCheckFirst) != 0) || !UseNewCode; + + bool new_val_maybe_null = ((ext_barrier_data & G1C2BarrierPostNotNull) != 0); + + __ block_comment(err_msg("barrier parts: gen_same_region %d gen_null_new %d gen_card_table %d maybe_null %d swap_same_null %d", gen_cross_region_check, gen_null_new_val_check, gen_card_table_check, new_val_maybe_null, null_check_first)); - // Crosses regions, storing null? - if (new_val_maybe_null) { - __ cmpptr(new_val, NULL_WORD); // new value == null? - __ jcc(Assembler::equal, done); + if (!null_check_first) { + if (gen_cross_region_check) { + generate_post_barrier_same_region_check(masm, store_addr, new_val, tmp1, done); + } + // Crosses regions, storing null? + if (gen_null_new_val_check && new_val_maybe_null) { + generate_post_barrier_null_new_value_check(masm, new_val, tmp1, done); + } + } else { + assert(gen_cross_region_check, "must be"); + assert(gen_null_new_val_check, "must be"); + generate_post_barrier_null_new_value_check(masm, new_val, tmp1, done); + generate_post_barrier_same_region_check(masm, store_addr, new_val, tmp1, done); } __ movptr(tmp1, store_addr); // tmp1 := store address @@ -344,7 +374,8 @@ static void generate_post_barrier_fast_path(MacroAssembler* masm, Address card_table_addr(thread, in_bytes(G1ThreadLocalData::card_table_base_offset())); __ addptr(tmp1, card_table_addr); // tmp1 := card address - if (UseCondCardMark) { + if (gen_card_table_check && UseCondCardMark) { + __ block_comment("card-table"); __ cmpb(Address(tmp1, 0), G1CardTable::clean_card_val()); // *(card address) == clean_card_val? __ jcc(Assembler::notEqual, done); } @@ -353,6 +384,14 @@ static void generate_post_barrier_fast_path(MacroAssembler* masm, __ movb(Address(tmp1, 0), G1CardTable::dirty_card_val()); // *(card address) := dirty_card_val } +static uint8_t gen_all_barrier_parts() { + return G1C2BarrierPostNotNull | G1C2BarrierPostGenCrossCheck | G1C2BarrierPostGenNullCheck | G1C2BarrierPostGenCardCheck; +} + +static uint8_t gen_no_barrier_parts() { + return 0; +} + void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, Register store_addr, Register new_val, @@ -360,7 +399,7 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, Register tmp, Register tmp2) { Label done; - generate_post_barrier_fast_path(masm, store_addr, new_val, thread, tmp, tmp2, done, true /* new_val_may_be_null */); + generate_post_barrier_fast_path(masm, store_addr, new_val, thread, tmp, tmp2, done, G1UseConcRefinement ? gen_all_barrier_parts() : gen_no_barrier_parts()); __ bind(done); } @@ -398,6 +437,8 @@ void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm, assert_different_registers(obj, pre_val, tmp); } + if (XXXSkipPreBarrier) return; + stub->initialize_registers(obj, pre_val, thread, tmp); generate_pre_barrier_fast_path(masm, thread); @@ -431,9 +472,9 @@ void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm, Register thread, Register tmp, Register tmp2, - bool new_val_maybe_null) { + uint8_t barrier_data) { Label done; - generate_post_barrier_fast_path(masm, store_addr, new_val, thread, tmp, tmp2, done, new_val_maybe_null); + generate_post_barrier_fast_path(masm, store_addr, new_val, thread, tmp, tmp2, done, barrier_data); __ bind(done); } @@ -529,6 +570,66 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier #undef __ +#define __ masm-> + +void G1BarrierSetAssembler::g1_write_barrier_post_profile_c1(ciMethodData* md, + int bci, + MacroAssembler* masm, + Register store_addr, + Register new_val, + Register thread, + Register tmp1, + Register tmp2) { + assert(md != nullptr, "must be"); + + ciProfileData* data = md->bci_to_data(bci); + assert(data != nullptr, "must be"); + if (!data->is_G1CounterData()) { + assert(!UseNewCode, "must be"); + return; + } + + + data = data->as_G1CounterData(); + assert_different_registers(store_addr, new_val, thread, tmp1, tmp2); + + Register mdp = tmp2; + __ mov_metadata(mdp, md->constant_encoding()); + __ increment(Address(mdp, md->byte_offset_of_slot(data, G1CounterData::visits_counter_offset()))); + + __ movptr(tmp1, new_val); + if (UseCompressedOops) { + __ decode_heap_oop(tmp1); + } + __ xorptr(tmp1, store_addr); + __ shrptr(tmp1, G1HeapRegion::LogOfHRGrainBytes); + __ setcc(Assembler::zero, tmp1); + __ addptr(Address(mdp, md->byte_offset_of_slot(data, G1CounterData::same_region_counter_offset())), tmp1); // How many same-region pointers + __ testptr(new_val, new_val); + __ setcc(Assembler::zero, tmp1); + __ addptr(Address(mdp, md->byte_offset_of_slot(data, G1CounterData::null_new_val_counter_offset())), tmp1); // How many zeros + + __ movptr(tmp1, Address(thread, in_bytes(G1ThreadLocalData::card_table_base_offset()))); + __ push(tmp2); + __ movptr(tmp2, store_addr); + __ shrptr(tmp2, G1CardTable::card_shift()); + __ cmpb(Address(tmp1, tmp2), G1CardTable::clean_card_val()); + __ setcc(Assembler::equal, tmp1); + __ pop(tmp2); + __ addptr(Address(mdp, md->byte_offset_of_slot(data, G1CounterData::clean_cards_counter_offset())), tmp1); // How many clean cards + + if (XXXDoYoungPreDirty) { + __ movptr(tmp1, Address(thread, in_bytes(G1ThreadLocalData::card_table_base_offset()))); + __ push(tmp2); + __ movptr(tmp2, store_addr); + __ shrptr(tmp2, G1CardTable::card_shift()); + __ cmpb(Address(tmp1, tmp2), G1CardTable::g1_young_card); + __ setcc(Assembler::equal, tmp1); + __ pop(tmp2); + __ addptr(Address(mdp, md->byte_offset_of_slot(data, G1CounterData::from_young_counter_offset())), tmp1); // How many from-young cards + } +} + void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm, Register store_addr, Register new_val, @@ -536,10 +637,12 @@ void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm, Register tmp1, Register tmp2) { Label done; - generate_post_barrier_fast_path(masm, store_addr, new_val, thread, tmp1, tmp2, done, true /* new_val_maybe_null */); - masm->bind(done); + generate_post_barrier_fast_path(masm, store_addr, new_val, thread, tmp1, tmp2, done, G1UseConcRefinement ? gen_all_barrier_parts() : gen_no_barrier_parts()); + __ bind(done); } +#undef __ + #define __ sasm-> void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { diff --git a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.hpp index 9b5a560ac2369..2bcc4924dfe1b 100644 --- a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.hpp @@ -28,11 +28,12 @@ #include "asm/macroAssembler.hpp" #include "gc/shared/modRefBarrierSetAssembler.hpp" -class LIR_Assembler; -class StubAssembler; +class ciMethodData; class G1PreBarrierStub; class G1BarrierStubC2; class G1PreBarrierStubC2; +class LIR_Assembler; +class StubAssembler; class G1BarrierSetAssembler: public ModRefBarrierSetAssembler { protected: @@ -66,6 +67,15 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler { void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); + void g1_write_barrier_post_profile_c1(ciMethodData* md, + int bci, + MacroAssembler* masm, + Register store_addr, + Register new_val, + Register thread, + Register tmp1, + Register tmp2); + void g1_write_barrier_post_c1(MacroAssembler* masm, Register store_addr, Register new_val, @@ -89,7 +99,7 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler { Register thread, Register tmp, Register tmp2, - bool new_val_maybe_null); + uint8_t barrier_data); #endif // COMPILER2 }; diff --git a/src/hotspot/cpu/x86/gc/g1/g1_x86_64.ad b/src/hotspot/cpu/x86/gc/g1/g1_x86_64.ad index fbf5bea1248a1..f3894ed907423 100644 --- a/src/hotspot/cpu/x86/gc/g1/g1_x86_64.ad +++ b/src/hotspot/cpu/x86/gc/g1/g1_x86_64.ad @@ -67,7 +67,7 @@ static void write_barrier_post(MacroAssembler* masm, Assembler::InlineSkippedInstructionsCounter skip_counter(masm); G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler()); bool new_val_maybe_null = G1BarrierStubC2::post_new_val_maybe_null(node); - g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, r15_thread, tmp1, tmp2, new_val_maybe_null); + g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, r15_thread, tmp1, tmp2, G1BarrierStubC2::ext_barrier_data(node)); } %} diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp index 3a3f01a640983..491b08f799bb8 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.cpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp @@ -568,7 +568,8 @@ void InterpreterMacroAssembler::load_resolved_klass_at_index(Register klass, // Kills: // rcx, rdi void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass, - Label& ok_is_subtype) { + Label& ok_is_subtype, + bool is_aastore) { assert(Rsub_klass != rax, "rax holds superklass"); LP64_ONLY(assert(Rsub_klass != r14, "r14 holds locals");) LP64_ONLY(assert(Rsub_klass != r13, "r13 holds bcp");) @@ -576,7 +577,7 @@ void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass, assert(Rsub_klass != rdi, "rdi holds 2ndary super array scan ptr"); // Profile the not-null value's klass. - profile_typecheck(rcx, Rsub_klass, rdi); // blows rcx, reloads rdi + profile_typecheck(rcx, Rsub_klass, rdi, is_aastore); // blows rcx, reloads rdi // Do the check. check_klass_subtype(Rsub_klass, rax, rcx, ok_is_subtype); // blows rcx @@ -1861,19 +1862,26 @@ void InterpreterMacroAssembler::profile_ret(Register return_bci, } -void InterpreterMacroAssembler::profile_null_seen(Register mdp) { +void InterpreterMacroAssembler::profile_null_seen(Register mdp, bool is_aastore) { if (ProfileInterpreter) { Label profile_continue; // If no method data exists, go to profile_continue. test_method_data_pointer(mdp, profile_continue); + if (UseNewCode && is_aastore) { + addptr(mdp, in_bytes(CombinedData::receiver_type_data_offset())); + } set_mdp_flag_at(mdp, BitData::null_seen_byte_constant()); // The method data pointer needs to be updated. int mdp_delta = in_bytes(BitData::bit_data_size()); if (TypeProfileCasts) { - mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); + if (UseNewCode && is_aastore) { + mdp_delta = in_bytes(CombinedData::receiver_type_data_size()); + } else { + mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); + } } update_mdp_by_constant(mdp, mdp_delta); @@ -1882,25 +1890,41 @@ void InterpreterMacroAssembler::profile_null_seen(Register mdp) { } -void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass, Register reg2) { +void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass, Register reg2, bool is_aastore) { if (ProfileInterpreter) { Label profile_continue; // If no method data exists, go to profile_continue. test_method_data_pointer(mdp, profile_continue); - // The method data pointer needs to be updated. - int mdp_delta = in_bytes(BitData::bit_data_size()); - if (TypeProfileCasts) { - mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); +// do no profiling for aastore here + if (TypeProfileCasts && UseNewCode && is_aastore) { + block_comment("aastore-do profiling {"); + + update_mdp_by_constant(mdp, in_bytes(CombinedData::receiver_type_data_offset())); - // Record the object type. record_klass_in_profile(klass, mdp, reg2, false); + + update_mdp_by_constant(mdp, in_bytes(CombinedData::receiver_type_data_size())); + NOT_LP64(assert(reg2 == rdi, "we know how to fix this blown reg");) NOT_LP64(restore_locals();) // Restore EDI - } - update_mdp_by_constant(mdp, mdp_delta); + block_comment("aastore-do profiling }"); + } else { + + // The method data pointer needs to be updated. + int mdp_delta = in_bytes(BitData::bit_data_size()); + if (TypeProfileCasts) { + mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); + + // Record the object type. + record_klass_in_profile(klass, mdp, reg2, false); + NOT_LP64(assert(reg2 == rdi, "we know how to fix this blown reg");) + NOT_LP64(restore_locals();) // Restore EDI + } + update_mdp_by_constant(mdp, mdp_delta); + } bind(profile_continue); } } @@ -1957,7 +1981,108 @@ void InterpreterMacroAssembler::profile_switch_case(Register index, } } +void InterpreterMacroAssembler::profile_oop_store(Address field, Register new_val) { + if (!UseG1GC) { + return; + } + if (!ProfileInterpreter) { + return; + } + + Label profile_continue; + + // rcx, rbx contain the field address + assert(field.base() == rcx, "should be"); + assert(field.index() == rbx, "should be"); + assert(field.disp() == 0, "should be"); + // rax the new_val + assert(new_val == rax, "should be"); + + block_comment("profile_oop_store {"); + + push(rax); + push(rbx); + push(rcx); + push(rdx); + Register addr = rcx; + // Register new_val = rax; + Register tmp = rbx; + Register mdp = rdx; + // If no method data exists, exit. + test_method_data_pointer(mdp, profile_continue); + + lea(addr, field); // now rbx is free as temp. + + if (UseCompressedOops) { + decode_heap_oop_not_null(new_val); + } + + movptr(tmp, addr); + xorptr(tmp, new_val); + shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); + setcc(Assembler::zero, tmp); + addptr(Address(mdp, in_bytes(G1CounterData::same_region_counter_offset())), tmp); + + testptr(new_val, new_val); + setcc(Assembler::zero, tmp); + addptr(Address(mdp, in_bytes(G1CounterData::null_new_val_counter_offset())), tmp); + + Register thread = LP64_ONLY(r15_thread) NOT_LP64(tmp); + + NOT_LP64(get_thread(tmp)); + movptr(tmp, Address(thread, G1ThreadLocalData::card_table_base_offset())); + shrptr(addr, G1CardTable::card_shift()); + cmpb(Address(tmp, addr), G1CardTable::clean_card_val()); + setcc(Assembler::equal, tmp); + addptr(Address(mdp, in_bytes(G1CounterData::clean_cards_counter_offset())), tmp); + + if (XXXDoYoungPreDirty) { + NOT_LP64(get_thread(tmp)); + movptr(tmp, Address(thread, G1ThreadLocalData::card_table_base_offset())); +// shrptr(addr, G1CardTable::card_shift()); + cmpb(Address(tmp, addr), G1CardTable::g1_young_card); + setcc(Assembler::equal, tmp); + addptr(Address(mdp, in_bytes(G1CounterData::from_young_counter_offset())), tmp); + } + + bind(profile_continue); + + pop(rdx); + pop(rcx); + pop(rbx); + pop(rax); + + block_comment("}"); +} + +void InterpreterMacroAssembler::profile_putfield_fix_mdp() { + if (!UseG1GC) { + return; + } + if (!ProfileInterpreter) { + return; + } + + Label profile_continue; + + Register mdp = rbx; + + block_comment("profile_putfield_fix_mdp {"); + + push(mdp); // Just in case. + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + addptr(Address(mdp, in_bytes(G1CounterData::visits_counter_offset())), 1); + update_mdp_by_constant(mdp, in_bytes(G1CounterData::counter_data_size())); + + bind(profile_continue); + pop(mdp); + + block_comment("}"); +} void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) { if (state == atos) { diff --git a/src/hotspot/cpu/x86/interp_masm_x86.hpp b/src/hotspot/cpu/x86/interp_masm_x86.hpp index 5d9a9071f8add..682668963ff15 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.hpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.hpp @@ -177,7 +177,7 @@ class InterpreterMacroAssembler: public MacroAssembler { // Generate a subtype check: branch to ok_is_subtype if sub_klass is // a subtype of super_klass. - void gen_subtype_check( Register sub_klass, Label &ok_is_subtype ); + void gen_subtype_check( Register sub_klass, Label &ok_is_subtype, bool is_aastore = false); // Dispatching void dispatch_prolog(TosState state, int step = 0); @@ -262,13 +262,16 @@ class InterpreterMacroAssembler: public MacroAssembler { Register scratch2, bool receiver_can_be_null = false); void profile_ret(Register return_bci, Register mdp); - void profile_null_seen(Register mdp); - void profile_typecheck(Register mdp, Register klass, Register scratch); + void profile_null_seen(Register mdp, bool is_aastore = false); + void profile_typecheck(Register mdp, Register klass, Register scratch, bool is_aastore = false); void profile_switch_default(Register mdp); void profile_switch_case(Register index_in_scratch, Register mdp, Register scratch2); + void profile_oop_store(Address field, Register new_val); + void profile_putfield_fix_mdp(); + // Debugging // only if +VerifyOops && state == atos #define interp_verify_oop(reg, state) _interp_verify_oop(reg, state, __FILE__, __LINE__); diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index a798dea08cc79..554b48974097b 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -4815,7 +4815,7 @@ Register MacroAssembler::allocate_if_noreg(Register r, return r; } -void MacroAssembler::check_klass_subtype_slow_path_table(Register sub_klass, + void MacroAssembler::check_klass_subtype_slow_path_table(Register sub_klass, Register super_klass, Register temp_reg, Register temp2_reg, diff --git a/src/hotspot/cpu/x86/templateTable_x86.cpp b/src/hotspot/cpu/x86/templateTable_x86.cpp index 441e4c8a0b877..c3441e7e7eb78 100644 --- a/src/hotspot/cpu/x86/templateTable_x86.cpp +++ b/src/hotspot/cpu/x86/templateTable_x86.cpp @@ -1148,7 +1148,7 @@ void TemplateTable::aastore() { // Generate subtype check. Blows rcx, rdi // Superklass in rax. Subklass in rbx. - __ gen_subtype_check(rbx, ok_is_subtype); + __ gen_subtype_check(rbx, ok_is_subtype, true /* is_aastore */); // Come here on failure // object is at TOS @@ -1158,15 +1158,15 @@ void TemplateTable::aastore() { __ bind(ok_is_subtype); // Get the value we will store - __ movptr(rax, at_tos()); - __ movl(rcx, at_tos_p1()); // index + __ movptr(rax, at_tos()); // reload value + __ movl(rcx, at_tos_p1()); // reload index // Now store using the appropriate barrier do_oop_store(_masm, element_address, rax, IS_ARRAY); __ jmp(done); // Have a null in rax, rdx=array, ecx=index. Store null at ary[idx] __ bind(is_null); - __ profile_null_seen(rbx); + __ profile_null_seen(rbx, true /* is_aastore */); // Store a null do_oop_store(_masm, element_address, noreg, IS_ARRAY); @@ -3245,6 +3245,8 @@ void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteContr putfield_or_static_helper(byte_no, is_static, rc, obj, off, tos_state); __ bind(Done); + + __ profile_putfield_fix_mdp(); } void TemplateTable::putfield_or_static_helper(int byte_no, bool is_static, RewriteControl rc, @@ -3299,6 +3301,7 @@ void TemplateTable::putfield_or_static_helper(int byte_no, bool is_static, Rewri __ pop(atos); if (!is_static) pop_and_check_object(obj); // Store into the field + __ profile_oop_store(field, rax); do_oop_store(_masm, field, rax); if (!is_static && rc == may_rewrite) { patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no); @@ -3517,6 +3520,8 @@ void TemplateTable::fast_storefield(TosState state) { fast_storefield_helper(field, rax); __ bind(Done); + + __ profile_putfield_fix_mdp(); } void TemplateTable::fast_storefield_helper(Address field, Register rax) { @@ -3524,6 +3529,7 @@ void TemplateTable::fast_storefield_helper(Address field, Register rax) { // access field switch (bytecode()) { case Bytecodes::_fast_aputfield: + __ profile_oop_store(field, rax); do_oop_store(_masm, field, rax); break; case Bytecodes::_fast_lputfield: diff --git a/src/hotspot/share/c1/c1_Canonicalizer.cpp b/src/hotspot/share/c1/c1_Canonicalizer.cpp index 87657038a4ce0..4d438e8c6fe2a 100644 --- a/src/hotspot/share/c1/c1_Canonicalizer.cpp +++ b/src/hotspot/share/c1/c1_Canonicalizer.cpp @@ -224,7 +224,8 @@ void Canonicalizer::do_StoreField (StoreField* x) { // limit this optimization to current block if (value != nullptr && in_current_block(conv)) { set_canonical(new StoreField(x->obj(), x->offset(), x->field(), value, x->is_static(), - x->state_before(), x->needs_patching())); + x->state_before(), x->needs_patching(), + x->profiled_method(), x->profiled_bci())); return; } } diff --git a/src/hotspot/share/c1/c1_GraphBuilder.cpp b/src/hotspot/share/c1/c1_GraphBuilder.cpp index 0179923bc306c..59197b36b9927 100644 --- a/src/hotspot/share/c1/c1_GraphBuilder.cpp +++ b/src/hotspot/share/c1/c1_GraphBuilder.cpp @@ -1790,7 +1790,13 @@ void GraphBuilder::access_field(Bytecodes::Code code) { Value mask = append(new Constant(new IntConstant(1))); val = append(new LogicOp(Bytecodes::_iand, val, mask)); } - append(new StoreField(append(obj), offset, field, val, true, state_before, needs_patching)); + ciMethod* profiled_method = nullptr; + int profiled_bci = 0; + if (field->type()->basic_type() == T_OBJECT) { + profiled_method = method(); + profiled_bci = bci(); + } + append(new StoreField(append(obj), offset, field, val, true, state_before, needs_patching, profiled_method, profiled_bci)); break; } case Bytecodes::_getfield: { @@ -1860,8 +1866,14 @@ void GraphBuilder::access_field(Bytecodes::Code code) { val = append(new LogicOp(Bytecodes::_iand, val, mask)); } StoreField* store = new StoreField(obj, offset, field, val, false, state_before, needs_patching); - if (!needs_patching) store = _memory->store(store); + if (!needs_patching) { + store = _memory->store(store); + } if (store != nullptr) { + if (field->type()->basic_type() == T_OBJECT) { + store->set_profiled_bci(bci()); + store->set_profiled_method(method()); + } append(store); } break; diff --git a/src/hotspot/share/c1/c1_Instruction.hpp b/src/hotspot/share/c1/c1_Instruction.hpp index e950afc981d19..a85435ba83b8e 100644 --- a/src/hotspot/share/c1/c1_Instruction.hpp +++ b/src/hotspot/share/c1/c1_Instruction.hpp @@ -829,13 +829,15 @@ LEAF(StoreField, AccessField) private: Value _value; + ciMethod* _profiled_method; + int _profiled_bci; + public: // creation StoreField(Value obj, int offset, ciField* field, Value value, bool is_static, - ValueStack* state_before, bool needs_patching) + ValueStack* state_before, bool needs_patching, ciMethod* profiled_method = nullptr, int profiled_bci = 0) : AccessField(obj, offset, field, is_static, state_before, needs_patching) - , _value(value) - { + , _value(value), _profiled_method(profiled_method), _profiled_bci(profiled_bci) { ASSERT_VALUES pin(); } @@ -843,6 +845,11 @@ LEAF(StoreField, AccessField) // accessors Value value() const { return _value; } + ciMethod* profiled_method() const { return _profiled_method; } + void set_profiled_method(ciMethod* value) { _profiled_method = value; } + int profiled_bci() const { return _profiled_bci; } + void set_profiled_bci(int value) { _profiled_bci = value; } + // generic virtual void input_values_do(ValueVisitor* f) { AccessField::input_values_do(f); f->visit(&_value); } }; diff --git a/src/hotspot/share/c1/c1_InstructionPrinter.cpp b/src/hotspot/share/c1/c1_InstructionPrinter.cpp index 5f865ae518d00..437559ff7014c 100644 --- a/src/hotspot/share/c1/c1_InstructionPrinter.cpp +++ b/src/hotspot/share/c1/c1_InstructionPrinter.cpp @@ -370,6 +370,7 @@ void InstructionPrinter::do_StoreField(StoreField* x) { print_value(x->value()); output()->print(" (%c)", type2char(x->field()->type()->basic_type())); output()->print(" %s", x->field()->name()->as_utf8()); + output()->print(" " PTR_FORMAT " (%d)", p2i(x->profiled_method()), x->profiled_bci()); } diff --git a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp index 74fdf7a5b76a3..5a1350b329bea 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.cpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp @@ -1684,7 +1684,8 @@ void LIRGenerator::do_StoreField(StoreField* x) { } access_store_at(decorators, field_type, object, LIR_OprFact::intConst(x->offset()), - value.result(), info != nullptr ? new CodeEmitInfo(info) : nullptr, info); + value.result(), info != nullptr ? new CodeEmitInfo(info) : nullptr, info, + x->profiled_method(), x->profiled_bci()); } void LIRGenerator::do_StoreIndexed(StoreIndexed* x) { @@ -1747,8 +1748,13 @@ void LIRGenerator::do_StoreIndexed(StoreIndexed* x) { decorators |= C1_MASK_BOOLEAN; } - access_store_at(decorators, x->elt_type(), array, index.result(), value.result(), - nullptr, null_check_info); + if (obj_store) { + access_store_at(decorators, x->elt_type(), array, index.result(), value.result(), + nullptr, null_check_info, x->profiled_method(), x->profiled_bci()); + } else { + access_store_at(decorators, x->elt_type(), array, index.result(), value.result(), + nullptr, null_check_info); + } } void LIRGenerator::access_load_at(DecoratorSet decorators, BasicType type, @@ -1777,9 +1783,10 @@ void LIRGenerator::access_load(DecoratorSet decorators, BasicType type, void LIRGenerator::access_store_at(DecoratorSet decorators, BasicType type, LIRItem& base, LIR_Opr offset, LIR_Opr value, - CodeEmitInfo* patch_info, CodeEmitInfo* store_emit_info) { + CodeEmitInfo* patch_info, CodeEmitInfo* store_emit_info, + ciMethod* profiled_method, int profiled_bci) { decorators |= ACCESS_WRITE; - LIRAccess access(this, decorators, base, offset, type, patch_info, store_emit_info); + LIRAccess access(this, decorators, base, offset, type, patch_info, store_emit_info, profiled_method, profiled_bci); if (access.is_raw()) { _barrier_set->BarrierSetC1::store_at(access, value); } else { diff --git a/src/hotspot/share/c1/c1_LIRGenerator.hpp b/src/hotspot/share/c1/c1_LIRGenerator.hpp index 518cd5fa5e724..8bee767375579 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.hpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.hpp @@ -293,7 +293,8 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure { public: void access_store_at(DecoratorSet decorators, BasicType type, LIRItem& base, LIR_Opr offset, LIR_Opr value, - CodeEmitInfo* patch_info = nullptr, CodeEmitInfo* store_emit_info = nullptr); + CodeEmitInfo* patch_info = nullptr, CodeEmitInfo* store_emit_info = nullptr, + ciMethod* profiled_method = nullptr, int profiled_bci = 0); void access_load_at(DecoratorSet decorators, BasicType type, LIRItem& base, LIR_Opr offset, LIR_Opr result, diff --git a/src/hotspot/share/ci/ciMethod.cpp b/src/hotspot/share/ci/ciMethod.cpp index 80277b91d2264..25bde0f806fd2 100644 --- a/src/hotspot/share/ci/ciMethod.cpp +++ b/src/hotspot/share/ci/ciMethod.cpp @@ -470,6 +470,7 @@ ciCallProfile ciMethod::call_profile_at_bci(int bci) { // Every profiled call site has a counter. int count = check_overflow(data->as_CounterData()->count(), java_code_at_bci(bci)); + assert(!data->is_CombinedData(), "must be3"); if (!data->is_ReceiverTypeData()) { result._receiver_count[0] = 0; // that's a definite zero } else { // ReceiverTypeData is a subclass of CounterData diff --git a/src/hotspot/share/ci/ciMethodData.cpp b/src/hotspot/share/ci/ciMethodData.cpp index 5abb342d03119..ef48b3c00e76c 100644 --- a/src/hotspot/share/ci/ciMethodData.cpp +++ b/src/hotspot/share/ci/ciMethodData.cpp @@ -303,6 +303,7 @@ bool ciMethodData::load_data() { void ciReceiverTypeData::translate_receiver_data_from(const ProfileData* data) { for (uint row = 0; row < row_limit(); row++) { + assert(!data->is_CombinedData(), "must be 6"); Klass* k = data->as_ReceiverTypeData()->receiver(row); if (k != nullptr) { if (k->is_loader_alive()) { @@ -367,6 +368,10 @@ ciProfileData* ciMethodData::data_from(DataLayout* data_layout) { return new ciBitData(data_layout); case DataLayout::counter_data_tag: return new ciCounterData(data_layout); + case DataLayout::g1counter_data_tag: + return new ciG1CounterData(data_layout); + case DataLayout::combined_data_tag: + return new ciCombinedData(data_layout); case DataLayout::jump_data_tag: return new ciJumpData(data_layout); case DataLayout::receiver_type_data_tag: @@ -805,7 +810,8 @@ void ciMethodData::dump_replay_data(outputStream* out) { dump_replay_data_call_type_helper(out, round, count, call_type_data); } } else if (pdata->is_ReceiverTypeData()) { - ciReceiverTypeData* vdata = (ciReceiverTypeData*)pdata; + guarantee(!pdata->is_CombinedData(), "must be 4"); + ciReceiverTypeData* vdata = (ciReceiverTypeData*)pdata->as_ReceiverTypeData(); dump_replay_data_receiver_type_helper(out, round, count, vdata); } else if (pdata->is_CallTypeData()) { ciCallTypeData* call_type_data = (ciCallTypeData*)pdata; @@ -883,86 +889,90 @@ void ciTypeEntries::print_ciklass(outputStream* st, intptr_t k) { } } -void ciTypeStackSlotEntries::print_data_on(outputStream* st) const { +void ciTypeStackSlotEntries::print_data_on(outputStream* st, bool cr) const { for (int i = 0; i < number_of_entries(); i++) { _pd->tab(st); st->print("%d: stack (%u) ", i, stack_slot(i)); print_ciklass(st, type(i)); - st->cr(); + if (cr) st->cr(); } } -void ciReturnTypeEntry::print_data_on(outputStream* st) const { +void ciReturnTypeEntry::print_data_on(outputStream* st, bool cr) const { _pd->tab(st); st->print("ret "); print_ciklass(st, type()); - st->cr(); + if (cr) st->cr(); } -void ciCallTypeData::print_data_on(outputStream* st, const char* extra) const { +void ciCallTypeData::print_data_on(outputStream* st, const char* extra, bool cr) const { print_shared(st, "ciCallTypeData", extra); if (has_arguments()) { tab(st, true); - st->print_cr("argument types"); - args()->print_data_on(st); + st->print("argument types "); + if (cr) st->cr(); + args()->print_data_on(st, cr); } if (has_return()) { tab(st, true); - st->print_cr("return type"); - ret()->print_data_on(st); + st->print("return type "); + if (cr) st->cr(); + ret()->print_data_on(st, cr); } } -void ciReceiverTypeData::print_receiver_data_on(outputStream* st) const { +void ciReceiverTypeData::print_receiver_data_on(outputStream* st, bool cr) const { uint row; int entries = 0; for (row = 0; row < row_limit(); row++) { if (receiver(row) != nullptr) entries++; } - st->print_cr("count(%u) entries(%u)", count(), entries); + st->print("count(%u) entries(%u) ", count(), entries); + if (cr) st->cr(); for (row = 0; row < row_limit(); row++) { if (receiver(row) != nullptr) { tab(st); receiver(row)->print_name_on(st); - st->print_cr("(%u)", receiver_count(row)); + st->print("(%u) ", receiver_count(row)); + if (cr) st->cr(); } } } -void ciReceiverTypeData::print_data_on(outputStream* st, const char* extra) const { +void ciReceiverTypeData::print_data_on(outputStream* st, const char* extra, bool cr) const { print_shared(st, "ciReceiverTypeData", extra); - print_receiver_data_on(st); + print_receiver_data_on(st, cr); } -void ciVirtualCallData::print_data_on(outputStream* st, const char* extra) const { +void ciVirtualCallData::print_data_on(outputStream* st, const char* extra, bool cr) const { print_shared(st, "ciVirtualCallData", extra); - rtd_super()->print_receiver_data_on(st); + rtd_super()->print_receiver_data_on(st, cr); } -void ciVirtualCallTypeData::print_data_on(outputStream* st, const char* extra) const { +void ciVirtualCallTypeData::print_data_on(outputStream* st, const char* extra, bool cr) const { print_shared(st, "ciVirtualCallTypeData", extra); rtd_super()->print_receiver_data_on(st); if (has_arguments()) { tab(st, true); st->print("argument types"); - args()->print_data_on(st); + args()->print_data_on(st, cr); } if (has_return()) { tab(st, true); st->print("return type"); - ret()->print_data_on(st); + ret()->print_data_on(st, cr); } } -void ciParametersTypeData::print_data_on(outputStream* st, const char* extra) const { +void ciParametersTypeData::print_data_on(outputStream* st, const char* extra, bool cr) const { st->print_cr("ciParametersTypeData"); - parameters()->print_data_on(st); + parameters()->print_data_on(st, cr); } -void ciSpeculativeTrapData::print_data_on(outputStream* st, const char* extra) const { +void ciSpeculativeTrapData::print_data_on(outputStream* st, const char* extra, bool cr) const { st->print_cr("ciSpeculativeTrapData"); tab(st); method()->print_short_name(st); - st->cr(); + if (cr) st->cr(); } #endif diff --git a/src/hotspot/share/ci/ciMethodData.hpp b/src/hotspot/share/ci/ciMethodData.hpp index a43d011b77ea2..32da7d7d66de5 100644 --- a/src/hotspot/share/ci/ciMethodData.hpp +++ b/src/hotspot/share/ci/ciMethodData.hpp @@ -59,6 +59,16 @@ class ciCounterData : public CounterData { ciCounterData(DataLayout* layout) : CounterData(layout) {}; }; +class ciG1CounterData : public G1CounterData { +public: + ciG1CounterData(DataLayout* layout) : G1CounterData(layout) {}; +}; + +class ciCombinedData : public CombinedData { +public: + ciCombinedData(DataLayout* layout) : CombinedData(layout) { } +}; + class ciJumpData : public JumpData { public: ciJumpData(DataLayout* layout) : JumpData(layout) {}; @@ -120,7 +130,7 @@ class ciTypeStackSlotEntries : public TypeStackSlotEntries, ciTypeEntries { } #ifndef PRODUCT - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, bool cr = true) const; #endif }; @@ -137,7 +147,7 @@ class ciReturnTypeEntry : public ReturnTypeEntry, ciTypeEntries { } #ifndef PRODUCT - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, bool cr = true) const; #endif }; @@ -186,7 +196,7 @@ class ciCallTypeData : public CallTypeData { } #ifndef PRODUCT - void print_data_on(outputStream* st, const char* extra = nullptr) const; + void print_data_on(outputStream* st, const char* extra = nullptr, bool cr = true) const; #endif }; @@ -213,8 +223,8 @@ class ciReceiverTypeData : public ReceiverTypeData { } void translate_receiver_data_from(const ProfileData* data); #ifndef PRODUCT - void print_data_on(outputStream* st, const char* extra = nullptr) const; - void print_receiver_data_on(outputStream* st) const; + void print_data_on(outputStream* st, const char* extra = nullptr, bool cr = true) const; + void print_receiver_data_on(outputStream* st, bool cr = true) const; #endif }; @@ -238,7 +248,7 @@ class ciVirtualCallData : public VirtualCallData { rtd_super()->translate_receiver_data_from(data); } #ifndef PRODUCT - void print_data_on(outputStream* st, const char* extra = nullptr) const; + void print_data_on(outputStream* st, const char* extra = nullptr, bool cr = true) const; #endif }; @@ -295,7 +305,7 @@ class ciVirtualCallTypeData : public VirtualCallTypeData { } #ifndef PRODUCT - void print_data_on(outputStream* st, const char* extra = nullptr) const; + void print_data_on(outputStream* st, const char* extra = nullptr, bool cr = true) const; #endif }; @@ -339,7 +349,7 @@ class ciParametersTypeData : public ParametersTypeData { } #ifndef PRODUCT - void print_data_on(outputStream* st, const char* extra = nullptr) const; + void print_data_on(outputStream* st, const char* extra = nullptr, bool cr = true) const; #endif }; @@ -358,7 +368,7 @@ class ciSpeculativeTrapData : public SpeculativeTrapData { } #ifndef PRODUCT - void print_data_on(outputStream* st, const char* extra = nullptr) const; + void print_data_on(outputStream* st, const char* extra = nullptr, bool cr = true) const; #endif }; diff --git a/src/hotspot/share/compiler/compilerDefinitions.cpp b/src/hotspot/share/compiler/compilerDefinitions.cpp index ad3d14012ff62..819a28ae68927 100644 --- a/src/hotspot/share/compiler/compilerDefinitions.cpp +++ b/src/hotspot/share/compiler/compilerDefinitions.cpp @@ -520,10 +520,12 @@ bool CompilerConfig::check_args_consistency(bool status) { FLAG_SET_CMDLINE(UseCompiler, false); } if (ProfileInterpreter) { + /* if (!FLAG_IS_DEFAULT(ProfileInterpreter)) { warning("ProfileInterpreter disabled due to -Xint."); } FLAG_SET_CMDLINE(ProfileInterpreter, false); + */ } if (TieredCompilation) { if (!FLAG_IS_DEFAULT(TieredCompilation)) { @@ -586,10 +588,12 @@ void CompilerConfig::ergo_initialize() { } if (ProfileInterpreter && CompilerConfig::is_c1_simple_only()) { + /* if (!FLAG_IS_DEFAULT(ProfileInterpreter)) { warning("ProfileInterpreter disabled due to client emulation mode"); } FLAG_SET_CMDLINE(ProfileInterpreter, false); + */ } #ifdef COMPILER2 diff --git a/src/hotspot/share/gc/g1/c1/g1BarrierSetC1.cpp b/src/hotspot/share/gc/g1/c1/g1BarrierSetC1.cpp index 5e918bf2bf58c..e69c5fabafe6c 100644 --- a/src/hotspot/share/gc/g1/c1/g1BarrierSetC1.cpp +++ b/src/hotspot/share/gc/g1/c1/g1BarrierSetC1.cpp @@ -43,6 +43,9 @@ #define __ gen->lir()-> #endif +#include "logging/log.hpp" +#include "logging/logStream.hpp" + void G1PreBarrierStub::emit_code(LIR_Assembler* ce) { G1BarrierSetAssembler* bs = (G1BarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); bs->gen_pre_barrier_stub(ce, this); @@ -53,6 +56,7 @@ void G1BarrierSetC1::pre_barrier(LIRAccess& access, LIR_Opr addr_opr, LIRGenerator* gen = access.gen(); DecoratorSet decorators = access.decorators(); + if (XXXSkipPreBarrier) return; // First we test whether marking is in progress. BasicType flag_type; bool patch = (decorators & C1_NEEDS_PATCHING) != 0; @@ -115,6 +119,43 @@ void G1BarrierSetC1::pre_barrier(LIRAccess& access, LIR_Opr addr_opr, __ branch_destination(slow->continuation()); } +static void print_profile_data(ciMethod* method, int bci, const char* text = "profiledata") { + if (method == nullptr) { + return; + } + + LogTarget(Debug, gc, barrier) lt; + if (!lt.is_enabled()) { + return; + } + LogStream ls(lt); + + ResourceMark rm; + + ciMethodData* md = method->method_data_or_null(); + if (UseNewCode && md != nullptr) { // Do not have profile data with -XX:-UseNewCode + ciProfileData* data = md->bci_to_data(bci); + if (data != nullptr) { + assert(data->is_G1CounterData(), "must be"); + G1CounterData* counter_data = data->as_G1CounterData(); + + ls.print_cr("C1 %s (%s): %s::%s @ %d - samples (%zu) same-region (%zu) null-new-val (%zu) clean-cards (%zu) from-young (%zu)", + text, + data->is_CombinedData() ? "aastore" : "putfield", + method->holder()->name()->as_utf8(), method->name()->as_utf8(), bci, + (size_t)counter_data->visits_count(), + (size_t)counter_data->same_region_count(), + (size_t)counter_data->null_new_val_count(), + (size_t)counter_data->clean_cards_count(), + (size_t)counter_data->from_young_count()); + + //counter_data->print_data_on(&ls); + } + } else { + ls.print_cr("C1 NO %s: %s@%d", text, method->name()->as_utf8(), bci); + } +} + class LIR_OpG1PostBarrier : public LIR_Op { friend class LIR_OpVisitState; @@ -124,19 +165,29 @@ class LIR_OpG1PostBarrier : public LIR_Op { LIR_Opr _thread; LIR_Opr _tmp1; LIR_Opr _tmp2; + LIR_Opr _tmp3; + + ciMethod* _method; + int _bci; public: LIR_OpG1PostBarrier(LIR_Opr addr, LIR_Opr new_val, LIR_Opr thread, LIR_Opr tmp1, - LIR_Opr tmp2) + LIR_Opr tmp2, + LIR_Opr tmp3, + ciMethod* method, + int bci) : LIR_Op(lir_none, lir_none, nullptr), _addr(addr), _new_val(new_val), _thread(thread), _tmp1(tmp1), - _tmp2(tmp2) + _tmp2(tmp2), + _tmp3(tmp3), + _method(method), + _bci(bci) {} virtual void visit(LIR_OpVisitState* state) { @@ -150,6 +201,7 @@ class LIR_OpG1PostBarrier : public LIR_Op { state->do_temp(_thread); state->do_temp(_tmp1); state->do_temp(_tmp2); + state->do_temp(_tmp3); if (_info != nullptr) { state->do_info(_info); @@ -166,17 +218,26 @@ class LIR_OpG1PostBarrier : public LIR_Op { Register thread = _thread->as_pointer_register(); Register tmp1 = _tmp1->as_pointer_register(); Register tmp2 = _tmp2->as_pointer_register(); - + Register tmp3 = _tmp3->as_pointer_register(); + // This may happen for a store of x.a = x - we do not need a post barrier for those // as the cross-region test will always exit early anyway. // The post barrier implementations can assume that addr and new_val are different // then. + // FIXME: maybe add to the flags? Should we profile anyway? if (addr == new_val) { ce->masm()->block_comment(err_msg("same addr/new_val due to self-referential store with imprecise card mark %s", addr->name())); return; } + print_profile_data(_method, _bci); + G1BarrierSetAssembler* bs_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler()); + if (_method != nullptr && + _method->method_data_or_null() != nullptr && + _method->method_data_or_null()->bci_to_data(_bci) != nullptr) { + bs_asm->g1_write_barrier_post_profile_c1(_method->method_data(), _bci, ce->masm(), addr, new_val, thread, tmp1, tmp2); + } bs_asm->g1_write_barrier_post_c1(ce->masm(), addr, new_val, thread, tmp1, tmp2); } @@ -186,6 +247,7 @@ class LIR_OpG1PostBarrier : public LIR_Op { _thread->print(out); out->print(" "); _tmp1->print(out); out->print(" "); _tmp2->print(out); out->print(" "); + _tmp3->print(out); out->print(" "); out->cr(); } @@ -204,6 +266,8 @@ void G1BarrierSetC1::post_barrier(LIRAccess& access, LIR_Opr addr, LIR_Opr new_v return; } + print_profile_data(access.profiled_method(), access.profiled_bci(), "profiledata before"); + // If the "new_val" is a constant null, no barrier is necessary. if (new_val->is_constant() && new_val->as_constant_ptr()->as_jobject() == nullptr) return; @@ -236,7 +300,10 @@ void G1BarrierSetC1::post_barrier(LIRAccess& access, LIR_Opr addr, LIR_Opr new_v new_val, gen->getThreadPointer() /* thread */, gen->new_pointer_register() /* tmp1 */, - gen->new_pointer_register() /* tmp2 */)); + gen->new_pointer_register() /* tmp2 */, + gen->new_pointer_register() /* tmp3 */, + access.profiled_method(), + access.profiled_bci())); } void G1BarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { diff --git a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp index cb77b5bddd6c3..62b877315671a 100644 --- a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp +++ b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp @@ -50,6 +50,10 @@ #include "utilities/growableArray.hpp" #include "utilities/macros.hpp" +#include "logging/log.hpp" +#include "logging/logTagSet.hpp" +#include "logging/logStream.hpp" + /* * Determine if the G1 pre-barrier can be removed. The pre-barrier is * required by SATB to make sure all objects live at the start of the @@ -293,15 +297,53 @@ uint G1BarrierSetC2::estimated_barrier_size(const Node* node) const { // static void write(MyObject obj1, Object o) { // obj1.o1 = o; // } + + // Approximate the number of nodes needed with the number of (x64) Assembly instructions + // in the fast path(!) since we do not have real nodes. + // + // From ZGC code: + // + // "A compare and branch corresponds to approximately four fast-path Ideal + // nodes (Cmp, Bool, If, If projection). The slow path (If projection and + // runtime call) is excluded since the corresponding code is laid out + // separately and does not directly affect performance." + + uint8_t barrier_data = MemNode::barrier_data(node); uint nodes = 0; if ((barrier_data & G1C2BarrierPre) != 0) { - nodes += 50; + + // cmpb/l marking-active-offset[tls_reg], 0 + // jz exit + nodes += XXXSkipPreBarrier ? 0 : 4; } if ((barrier_data & G1C2BarrierPost) != 0) { - // Approximate the number of nodes needed with the number of Assembly instructions + // Approximate the number of nodes needed with the number of (x64) Assembly instructions // since we do not have real nodes. - nodes += 12; + + // mov card_base_addr, card-table-base-ofs[tls_reg] + // shr addr, card_shift + // add addr, card_base_addr + // mov [addr], dirty_card_val + nodes += 4; + if ((barrier_data & G1C2BarrierPostGenNullCheck) != 0) { + // cmpb new_val, 0 + // je exit + nodes += 4; + } + if ((barrier_data & G1C2BarrierPostGenCrossCheck) != 0) { + // FIXME: add uncompressing of new_val too with compressed oops? + // [mov tmp, new_val] + // xor tmp, addr + // shr tmp, + // jz exit // ("cmp + if") + nodes += 6; + } + if ((barrier_data & G1C2BarrierPostGenCardCheck) != 0) { + // cmpb card[addr >> card_shift], clean_card_val + // jne exit + nodes += 4; + } } return nodes; } @@ -312,7 +354,7 @@ bool G1BarrierSetC2::can_initialize_object(const StoreNode* store) const { // if it does not have any barrier, or if it has barriers that can be safely // elided (because of the compensation steps taken on the allocation slow path // when ReduceInitialCardMarks is enabled). - return (MemNode::barrier_data(store) == 0) || use_ReduceInitialCardMarks(); + return (MemNode::barrier_data(store) == 0) || use_ReduceInitialCardMarks(); // FIXME :( } void G1BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const { @@ -323,6 +365,116 @@ void G1BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* BarrierSetC2::clone_at_expansion(phase, ac); } +template +static T saturated_sub(T x, T y) { + return (x < y) ? T() : (x - y); +} + +static uint8_t barrier_data_from_profile(C2Access& access) { + LogTarget(Debug, gc, barrier) lt; + + ciProfileData* profile = nullptr; + if (access.is_parse_access()) { + C2ParseAccess& pa = static_cast(access); + ciMethod* method = pa.kit()->method(); + profile = method->method_data()->bci_to_data(pa.kit()->bci()); + } + + uint8_t use_all_filters = G1C2BarrierPostGenNullCheck | G1C2BarrierPostGenCrossCheck | G1C2BarrierPostGenCardCheck; + uint8_t use_no_filters = 0; + + if (profile == nullptr || !profile->is_G1CounterData()) { + C2ParseAccess& pa = static_cast(access); + ciMethod* method = pa.kit()->method(); + + LogStream ls(lt); + if (method != nullptr) { + ls.print("C2 NO profile: "); + char buf[1024]; + Thread::current()->as_Compiler_thread()->env()->task()->print_line_on_error(&ls, buf, 1024); + // FIXME: below crashes often for one or the other reason... :( + //method->print_short_name(&ls); + /* + method->holder()->name()->get_symbol() == nullptr ? "{unknown}" : method->holder()->name()->as_utf8(), + !Symbol::is_valid(method->name()->get_symbol()) ? "{unknown}" : method->name()->as_utf8(), + pa.kit()->bci(), profile == nullptr ? "n/a" : BOOL_TO_STR(profile->is_G1CounterData())); + */ + ls.print(" @ %d", pa.kit()->bci()); + } else { + ls.print("C2 NO profile: (unknown method) @ %d", pa.kit()->bci()); + } + ls.cr(); + + return UseNewCode3 ? use_no_filters : use_all_filters; + } + G1CounterData* data = profile->as_G1CounterData(); + + uint8_t result = 0; + bool too_few_samples = data->visits_count() <= 10; + bool null_check_first = false; + + double same_region_ratio = 0.0; + double null_new_val_ratio = 0.0; + double not_clean_card_ratio = 0.0; + double from_young_ratio = 0.0; + + if (UseNewCode3) { + result = use_no_filters; + } else if (too_few_samples) { // Too few samples. + result = use_all_filters; + } else { + + same_region_ratio = clamp_unit((double)data->same_region_count() / data->visits_count()); + null_new_val_ratio = clamp_unit((double)data->null_new_val_count() / data->visits_count()); + not_clean_card_ratio = 1.0 - clamp_unit((double)data->clean_cards_count() / data->visits_count()); + from_young_ratio = clamp_unit((double)data->from_young_count() / data->visits_count()); + + if (same_region_ratio > 0.1) { + result |= G1C2BarrierPostGenCrossCheck; + } + if (null_new_val_ratio > 0.1) { + result |= G1C2BarrierPostGenNullCheck; + } + if (not_clean_card_ratio > 0.1) { + result |= G1C2BarrierPostGenCardCheck; + } + uint8_t cross_null_check_flags = G1C2BarrierPostGenCrossCheck | G1C2BarrierPostGenNullCheck; + if ((result & cross_null_check_flags) == cross_null_check_flags) { + if (null_new_val_ratio > same_region_ratio) { + result |= G1C2BarrierPostNullCheckFirst; + null_check_first = true; + } + } + if (from_young_ratio > 0.9) { + result = use_no_filters; + } + } + + C2ParseAccess& pa = static_cast(access); + ciMethod* method = pa.kit()->method(); + + if (lt.is_enabled()) { + LogStream ls(lt); + + ResourceMark rm; + + ls.print_cr("C2 profile (%s): %s::%s @ %d - few-samples %s (%zu) same-region %s (%zu %.2f) null-new-val %s (%zu %.2f) not-clean-card %s (%zu %.2f) from-young %s (%zu %.2f) null-first %s", + profile->is_CombinedData() ? "aastore" : "putfield", + method->holder()->name()->as_utf8(), method->name()->as_utf8(), + pa.kit()->bci(), + BOOL_TO_STR(too_few_samples), data->visits_count(), + BOOL_TO_STR((result & G1C2BarrierPostGenCrossCheck) != 0), data->same_region_count(), same_region_ratio, + BOOL_TO_STR((result & G1C2BarrierPostGenNullCheck) != 0), data->null_new_val_count(), null_new_val_ratio, + BOOL_TO_STR((result & G1C2BarrierPostGenCardCheck) != 0), saturated_sub(data->visits_count(), data->clean_cards_count()), not_clean_card_ratio, + BOOL_TO_STR(from_young_ratio > 0.9), data->from_young_count(), from_young_ratio, + BOOL_TO_STR(null_check_first) + ); + profile->print_data_on(&ls); + } + + return result; +} + Node* G1BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { DecoratorSet decorators = access.decorators(); bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; @@ -343,35 +495,36 @@ Node* G1BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) co // No keep-alive means no need for the pre-barrier. access.set_barrier_data(access.barrier_data() & ~G1C2BarrierPre); } + access.set_ext_barrier_data(barrier_data_from_profile(access)); return BarrierSetC2::store_at_resolved(access, val); } Node* G1BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { - GraphKit* kit = access.kit(); if (!access.is_oop()) { return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); } access.set_barrier_data(G1C2BarrierPre | G1C2BarrierPost); + access.set_ext_barrier_data(barrier_data_from_profile(access)); return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); } Node* G1BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { - GraphKit* kit = access.kit(); if (!access.is_oop()) { return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); } access.set_barrier_data(G1C2BarrierPre | G1C2BarrierPost); + access.set_ext_barrier_data(barrier_data_from_profile(access)); return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); } Node* G1BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { - GraphKit* kit = access.kit(); if (!access.is_oop()) { return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, value_type); } access.set_barrier_data(G1C2BarrierPre | G1C2BarrierPost); + access.set_ext_barrier_data(barrier_data_from_profile(access)); return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, value_type); } @@ -416,6 +569,15 @@ bool G1BarrierStubC2::post_new_val_maybe_null(const MachNode* node) { return (node->barrier_data() & G1C2BarrierPostNotNull) == 0; } +uint8_t G1BarrierStubC2::barrier_data(const MachNode* node) { + ShouldNotReachHere(); // FIXME: remove method? + return node->barrier_data(); +} + +uint8_t G1BarrierStubC2::ext_barrier_data(const MachNode* node) { + return node->ext_barrier_data(); +} + G1PreBarrierStubC2::G1PreBarrierStubC2(const MachNode* node) : G1BarrierStubC2(node) {} bool G1PreBarrierStubC2::needs_barrier(const MachNode* node) { @@ -531,5 +693,15 @@ void G1BarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) c if ((mach->barrier_data() & G1C2BarrierPostNotNull) != 0) { st->print("notnull "); } + /* // FIXME: tests will fail with these additional strings :( + if ((mach->ext_barrier_data() & G1C2BarrierPostGenCrossCheck) != 0) { + st->print("same-check "); + } + if ((mach->ext_barrier_data() & G1C2BarrierPostGenNullCheck) != 0) { + st->print("not-null-check"); + } + if ((mach->ext_barrier_data() & G1C2BarrierPostGenCardCheck) != 0) { + st->print("card-check"); + }*/ } #endif // !PRODUCT diff --git a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp index ec984a2cc35b9..c9279e4458115 100644 --- a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp +++ b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp @@ -34,6 +34,10 @@ class TypeFunc; const int G1C2BarrierPre = 1; const int G1C2BarrierPost = 2; const int G1C2BarrierPostNotNull = 4; +const int G1C2BarrierPostGenCrossCheck = 8; +const int G1C2BarrierPostGenNullCheck = 16; +const int G1C2BarrierPostGenCardCheck = 32; +const int G1C2BarrierPostNullCheckFirst = 64; // Probably not worth class G1BarrierStubC2 : public BarrierStubC2 { public: @@ -41,6 +45,9 @@ class G1BarrierStubC2 : public BarrierStubC2 { static bool needs_post_barrier(const MachNode* node); static bool post_new_val_maybe_null(const MachNode* node); + static uint8_t barrier_data(const MachNode* node); + static uint8_t ext_barrier_data(const MachNode* node); + G1BarrierStubC2(const MachNode* node); virtual void emit_code(MacroAssembler& masm) = 0; }; diff --git a/src/hotspot/share/gc/g1/g1Allocator.cpp b/src/hotspot/share/gc/g1/g1Allocator.cpp index d72c26f2359aa..be8c39ba440b2 100644 --- a/src/hotspot/share/gc/g1/g1Allocator.cpp +++ b/src/hotspot/share/gc/g1/g1Allocator.cpp @@ -263,6 +263,9 @@ HeapWord* G1Allocator::survivor_attempt_allocation(size_t min_word_size, } } } + if (result != nullptr) { + _g1h->dirty_young_block(result, *actual_word_size); + } return result; } diff --git a/src/hotspot/share/gc/g1/g1CardTable.hpp b/src/hotspot/share/gc/g1/g1CardTable.hpp index 79f31db2a330f..8dd736a693083 100644 --- a/src/hotspot/share/gc/g1/g1CardTable.hpp +++ b/src/hotspot/share/gc/g1/g1CardTable.hpp @@ -73,7 +73,8 @@ class G1CardTable : public CardTable { g1_dirty_card = dirty_card, g1_card_already_scanned = 0x1, g1_to_cset_card = 0x2, - g1_from_remset_card = 0x4 + g1_from_remset_card = 0x4, + g1_young_card = 0xf0 }; static const size_t WordAllClean = SIZE_MAX; diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp index cc4a7633fd3ea..39fbaf1f90aad 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp @@ -606,6 +606,7 @@ inline HeapWord* G1CollectedHeap::attempt_allocation(size_t min_word_size, assert_heap_not_locked(); if (result != nullptr) { + dirty_young_block(result, *actual_word_size); assert(*actual_word_size != 0, "Actual size must have been set here"); } else { *actual_word_size = 0; diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp index d8a765093bf37..14609dde64233 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp @@ -699,6 +699,11 @@ class G1CollectedHeap : public CollectedHeap { // Add the given region to the retained regions collection set candidates. void retain_region(G1HeapRegion* hr); + // It dirties the cards that cover the block so that the post + // write barrier never queues anything when updating objects on this + // block. It is assumed (and in fact we assert) that the block + // belongs to a young region. + inline void dirty_young_block(HeapWord* start, size_t word_size); // Frees a humongous region by collapsing it into individual regions // and calling free_region() for each of them. The freed regions // will be added to the free list that's passed as a parameter (this diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp index 0bd346082cfcf..6132fa5c991b6 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp @@ -257,6 +257,30 @@ inline void G1CollectedHeap::unpin_object(JavaThread* thread, oop obj) { G1ThreadLocalData::pin_count_cache(thread).dec_count(obj_region_idx); } +// It dirties the cards that cover the block so that the post +// write barrier never queues anything when updating objects on this +// block. It is assumed (and in fact we assert) that the block +// belongs to a young region. +inline void G1CollectedHeap::dirty_young_block(HeapWord* start, size_t word_size) { + assert_heap_not_locked(); + if (!XXXDoYoungPreDirty) { return; } + + // Assign the containing region to containing_hr so that we don't + // have to keep calling heap_region_containing() in the + // asserts below. + DEBUG_ONLY(G1HeapRegion* containing_hr = heap_region_containing(start);) + assert(word_size > 0, "pre-condition"); + assert(containing_hr->is_in(start), "it should contain start"); + assert(containing_hr->is_young(), "it should be young"); + assert(!containing_hr->is_humongous(), "it should not be humongous"); + + HeapWord* end = start + word_size; + assert(containing_hr->is_in(end - 1), "it should also contain end - 1"); + + MemRegion mr(start, end); + card_table()->dirty_MemRegion(mr, G1CardTable::g1_young_card); +} + inline bool G1CollectedHeap::is_obj_dead(const oop obj) const { assert(obj != nullptr, "precondition"); diff --git a/src/hotspot/share/gc/g1/g1HeapVerifier.cpp b/src/hotspot/share/gc/g1/g1HeapVerifier.cpp index a27bccfc2ced5..5afd7a82eb31c 100644 --- a/src/hotspot/share/gc/g1/g1HeapVerifier.cpp +++ b/src/hotspot/share/gc/g1/g1HeapVerifier.cpp @@ -562,10 +562,11 @@ class G1VerifyCardTableCleanup: public G1HeapRegionClosure { G1VerifyCardTableCleanup(G1HeapVerifier* verifier) : _verifier(verifier) { } virtual bool do_heap_region(G1HeapRegion* r) { - _verifier->verify_ct_clean_region(r); if (r->is_survivor()) { + _verifier->verify_ct_young_region(r); _verifier->verify_rt_clean_region(r); } else { + _verifier->verify_ct_clean_region(r); _verifier->verify_rt_clean_from_top(r); } return false; @@ -616,6 +617,16 @@ void G1HeapVerifier::verify_rt_dirty_to_dummy_top(G1HeapRegion* hr) { ct->verify_dirty_region(mr); } +void G1HeapVerifier::verify_ct_young_region(G1HeapRegion* hr) { + if (XXXDoYoungPreDirty && hr->is_young()) { + G1CardTable* ct = _g1h->card_table(); + MemRegion mr(hr->bottom(), hr->end()); + ct->verify_region(mr, G1CardTable::dirty_card_val(), false); // Dirty OR young. + } else { + verify_ct_clean_region(hr); + } +} + void G1HeapVerifier::verify_ct_clean_region(G1HeapRegion* hr) { G1CardTable* ct = _g1h->card_table(); MemRegion mr(hr->bottom(), hr->end()); diff --git a/src/hotspot/share/gc/g1/g1HeapVerifier.hpp b/src/hotspot/share/gc/g1/g1HeapVerifier.hpp index a4e1148f11bbb..ebbae64ee04a0 100644 --- a/src/hotspot/share/gc/g1/g1HeapVerifier.hpp +++ b/src/hotspot/share/gc/g1/g1HeapVerifier.hpp @@ -81,6 +81,7 @@ class G1HeapVerifier : public CHeapObj { void verify_card_table_cleanup(); void verify_refinement_table_clean(); + void verify_ct_young_region(G1HeapRegion* hr); void verify_ct_clean_region(G1HeapRegion* hr); void verify_rt_dirty_to_dummy_top(G1HeapRegion* hr); void verify_rt_clean_from_top(G1HeapRegion* hr); diff --git a/src/hotspot/share/gc/g1/g1RemSet.cpp b/src/hotspot/share/gc/g1/g1RemSet.cpp index 65c083da5205c..6c2a47c5aa40d 100644 --- a/src/hotspot/share/gc/g1/g1RemSet.cpp +++ b/src/hotspot/share/gc/g1/g1RemSet.cpp @@ -1085,9 +1085,15 @@ class G1MergeHeapRootsTask : public WorkerTask { // make sure the refinement table is clean for all regions either in // concurrent refinement or in the merge refinement table phase earlier. if (!_initial_evacuation) { + assert(!hr->is_young(), "should not be young regions outside of initial evacuation"); hr->clear_refinement_table(); } else { - assert_refinement_table_clear(hr); + if (XXXDoYoungPreDirty && hr->is_young()) { + // Young gen needs to be cleared. + hr->clear_refinement_table(); + } else { + assert_refinement_table_clear(hr); + } } // Evacuation failure uses the bitmap to record evacuation failed objects, // so the bitmap for the regions in the collection set must be cleared if not already. diff --git a/src/hotspot/share/gc/g1/g1_globals.hpp b/src/hotspot/share/gc/g1/g1_globals.hpp index ed02ba2dc5cad..27c388d6b64cb 100644 --- a/src/hotspot/share/gc/g1/g1_globals.hpp +++ b/src/hotspot/share/gc/g1/g1_globals.hpp @@ -335,6 +335,12 @@ "scan cost related prediction samples. A sample must involve " \ "the same or more than this number of code roots to be used.") \ \ + product(bool, XXXSkipPreBarrier, false, \ + "Skip pre-barrier generation") \ + \ + product(bool, XXXDoYoungPreDirty, false, \ + "Pre-Dirty young gen. Allows in-young-gen profiling.") \ + \ GC_G1_EVACUATION_FAILURE_FLAGS(develop, \ develop_pd, \ product, \ diff --git a/src/hotspot/share/gc/shared/c1/barrierSetC1.hpp b/src/hotspot/share/gc/shared/c1/barrierSetC1.hpp index 0fd6493d50e3f..0b1c1c8c94ca1 100644 --- a/src/hotspot/share/gc/shared/c1/barrierSetC1.hpp +++ b/src/hotspot/share/gc/shared/c1/barrierSetC1.hpp @@ -70,11 +70,14 @@ class LIRAccess: public StackObj { LIR_Opr _resolved_addr; CodeEmitInfo* _patch_emit_info; CodeEmitInfo* _access_emit_info; + ciMethod* _profiled_method; + int _profiled_bci; public: LIRAccess(LIRGenerator* gen, DecoratorSet decorators, LIRAddressOpr base, LIRAddressOpr offset, BasicType type, - CodeEmitInfo* patch_emit_info = nullptr, CodeEmitInfo* access_emit_info = nullptr) : + CodeEmitInfo* patch_emit_info = nullptr, CodeEmitInfo* access_emit_info = nullptr, + ciMethod* profiled_method = nullptr, int profiled_bci = 0) : _gen(gen), _decorators(AccessInternal::decorator_fixup(decorators, type)), _base(base), @@ -82,7 +85,9 @@ class LIRAccess: public StackObj { _type(type), _resolved_addr(), _patch_emit_info(patch_emit_info), - _access_emit_info(access_emit_info) {} + _access_emit_info(access_emit_info), + _profiled_method(profiled_method), + _profiled_bci(profiled_bci) {} void load_base() { _base.item().load_item(); } void load_offset() { _offset.item().load_nonconstant(); } @@ -95,6 +100,8 @@ class LIRAccess: public StackObj { LIRGenerator* gen() const { return _gen; } CodeEmitInfo*& patch_emit_info() { return _patch_emit_info; } CodeEmitInfo*& access_emit_info() { return _access_emit_info; } + ciMethod* profiled_method() { return _profiled_method; } + int profiled_bci() { return _profiled_bci; } LIRAddressOpr& base() { return _base; } LIRAddressOpr& offset() { return _offset; } BasicType type() const { return _type; } diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp index 080af1c9693ed..402e0b6599563 100644 --- a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp +++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp @@ -109,10 +109,6 @@ Label* BarrierStubC2::continuation() { return &_continuation; } -uint8_t BarrierStubC2::barrier_data() const { - return _node->barrier_data(); -} - void BarrierStubC2::preserve(Register r) { const VMReg vm_reg = r->as_VMReg(); assert(vm_reg->is_Register(), "r must be a general-purpose register"); @@ -157,7 +153,7 @@ Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) cons store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), bt, mo, requires_atomic_access, unaligned, mismatched, - unsafe, access.barrier_data()); + unsafe, access.barrier_data(), access.ext_barrier_data()); } else { assert(access.is_opt_access(), "either parse or opt access"); C2OptAccess& opt_access = static_cast(access); @@ -176,6 +172,7 @@ Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) cons st->set_mismatched_access(); } st->set_barrier_data(access.barrier_data()); + st->set_ext_barrier_data(access.ext_barrier_data()); store = gvn.transform(st); if (store == st) { mm->set_memory_at(alias, st); diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp index a78fd434ad94e..414bcb787ebd0 100644 --- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp +++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp @@ -107,6 +107,7 @@ class C2Access: public StackObj { Node* _raw_access; BasicType _type; uint8_t _barrier_data; + uint8_t _ext_barrier_data; void fixup_decorators(); @@ -118,7 +119,8 @@ class C2Access: public StackObj { _addr(addr), _raw_access(nullptr), _type(type), - _barrier_data(0) + _barrier_data(0), + _ext_barrier_data(0) {} DecoratorSet decorators() const { return _decorators; } @@ -131,6 +133,8 @@ class C2Access: public StackObj { uint8_t barrier_data() const { return _barrier_data; } void set_barrier_data(uint8_t data) { _barrier_data = data; } + uint8_t ext_barrier_data() const { return _ext_barrier_data; } + void set_ext_barrier_data(uint8_t data) { _ext_barrier_data = data; } void set_raw_access(Node* raw_access) { _raw_access = raw_access; } virtual void set_memory() {} // no-op for normal accesses, but not for atomic accesses. @@ -254,8 +258,6 @@ class BarrierStubC2 : public ArenaObj { Label* entry(); // Return point from the stub (typically end of barrier). Label* continuation(); - // High-level, GC-specific barrier flags. - uint8_t barrier_data() const; // Preserve the value in reg across runtime calls in this barrier. void preserve(Register reg); diff --git a/src/hotspot/share/gc/shared/cardTable.cpp b/src/hotspot/share/gc/shared/cardTable.cpp index f1200a12dfb54..1309cfaf91f5d 100644 --- a/src/hotspot/share/gc/shared/cardTable.cpp +++ b/src/hotspot/share/gc/shared/cardTable.cpp @@ -199,13 +199,13 @@ void CardTable::resize_covered_region(MemRegion new_region) { // Note that these versions are precise! The scanning code has to handle the // fact that the write barrier may be either precise or imprecise. -void CardTable::dirty_MemRegion(MemRegion mr) { +void CardTable::dirty_MemRegion(MemRegion mr, CardValue value) { assert(align_down(mr.start(), HeapWordSize) == mr.start(), "Unaligned start"); assert(align_up (mr.end(), HeapWordSize) == mr.end(), "Unaligned end" ); assert(_covered[0].contains(mr) || _covered[1].contains(mr), "precondition"); CardValue* cur = byte_for(mr.start()); CardValue* last = byte_after(mr.last()); - memset(cur, dirty_card, pointer_delta(last, cur, sizeof(CardValue))); + memset(cur, value, pointer_delta(last, cur, sizeof(CardValue))); } void CardTable::clear_MemRegion(MemRegion mr) { diff --git a/src/hotspot/share/gc/shared/cardTable.hpp b/src/hotspot/share/gc/shared/cardTable.hpp index 55dc0f369b58c..ca187ef93022f 100644 --- a/src/hotspot/share/gc/shared/cardTable.hpp +++ b/src/hotspot/share/gc/shared/cardTable.hpp @@ -101,7 +101,7 @@ class CardTable: public CHeapObj { // Dirty the bytes corresponding to "mr" (not all of which must be // covered.) - void dirty_MemRegion(MemRegion mr); + void dirty_MemRegion(MemRegion mr, CardValue value = dirty_card); // Clear (to clean_card) the bytes entirely contained within "mr" (not // all of which must be covered.) diff --git a/src/hotspot/share/interpreter/bytecodeTracer.cpp b/src/hotspot/share/interpreter/bytecodeTracer.cpp index cdb53b62f8c40..4907ad4361e0b 100644 --- a/src/hotspot/share/interpreter/bytecodeTracer.cpp +++ b/src/hotspot/share/interpreter/bytecodeTracer.cpp @@ -119,6 +119,9 @@ class BytecodePrinter { } else { code = Bytecodes::code_at(method(), bcp); } + if (UseNewCode2 && code != Bytecodes::_aastore && code != Bytecodes::_putfield && code != Bytecodes::_putstatic) { + return; + } _code = code; _next_pc = is_wide() ? bcp+2 : bcp+1; // Trace each bytecode unless we're truncating the tracing output, then only print the first @@ -136,6 +139,23 @@ class BytecodePrinter { st->print("%8d %4d %s", BytecodeCounter::counter_value(), bci, Bytecodes::name(code)); } + + MethodData* md = method->method_data(); + if (md != nullptr) { + MutexLocker ml(md->extra_data_lock(), Mutex::_no_safepoint_check_flag); + + ProfileData* data = md->bci_to_data(bci); + if (data != nullptr) { + st->print("profile data: "); + data->print_data_on(st, nullptr, false); + st->print(" "); + } else { + st->print(" NO profile data "); + } + } else { + st->print(" NO method data "); + } + print_attributes(bci, st); } // Set is_wide for the next one, since the caller of this doesn't skip @@ -185,7 +205,8 @@ static BytecodePrinter _interpreter_printer; void BytecodeTracer::trace_interpreter(const methodHandle& method, address bcp, uintptr_t tos, uintptr_t tos2, outputStream* st) { if (TraceBytecodes && BytecodeCounter::counter_value() >= TraceBytecodesAt) { - ttyLocker ttyl; // 5065316: keep the following output coherent + MutexLocker x(ByteCodeTracer_lock, Mutex::_no_safepoint_check_flag); + // 5065316: keep the following output coherent // The ttyLocker also prevents races between two threads // trying to use the single instance of BytecodePrinter. // diff --git a/src/hotspot/share/interpreter/interpreterRuntime.cpp b/src/hotspot/share/interpreter/interpreterRuntime.cpp index cb85513eed0ff..5e0ec348ca534 100644 --- a/src/hotspot/share/interpreter/interpreterRuntime.cpp +++ b/src/hotspot/share/interpreter/interpreterRuntime.cpp @@ -1548,7 +1548,11 @@ JRT_LEAF(intptr_t, InterpreterRuntime::trace_bytecode(JavaThread* current, intpt LastFrameAccessor last_frame(current); assert(last_frame.is_interpreted_frame(), "must be an interpreted frame"); methodHandle mh(current, last_frame.method()); - BytecodeTracer::trace_interpreter(mh, last_frame.bcp(), tos, tos2); + LogTarget(Trace, gc, barrier) lt; + if (lt.is_enabled()) { + LogStream ls(lt); + BytecodeTracer::trace_interpreter(mh, last_frame.bcp(), tos, tos2, &ls); + } return preserve_this_value; JRT_END #endif // !PRODUCT diff --git a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp index a3d5046975f58..d42ad7636018b 100644 --- a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp +++ b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp @@ -673,6 +673,7 @@ declare_constant(DataLayout::no_tag) \ declare_constant(DataLayout::bit_data_tag) \ declare_constant(DataLayout::counter_data_tag) \ + declare_constant(DataLayout::g1counter_data_tag) \ declare_constant(DataLayout::jump_data_tag) \ declare_constant(DataLayout::receiver_type_data_tag) \ declare_constant(DataLayout::virtual_call_data_tag) \ diff --git a/src/hotspot/share/oops/methodData.cpp b/src/hotspot/share/oops/methodData.cpp index 3c0c0b5468971..b53ff8cfc75eb 100644 --- a/src/hotspot/share/oops/methodData.cpp +++ b/src/hotspot/share/oops/methodData.cpp @@ -161,9 +161,9 @@ void ProfileData::tab(outputStream* st, bool first) const { // whether a checkcast bytecode has seen a null value. -void BitData::print_data_on(outputStream* st, const char* extra) const { +void BitData::print_data_on(outputStream* st, const char* extra, bool cr) const { print_shared(st, "BitData", extra); - st->cr(); + if (cr) st->cr(); } // ================================================================== @@ -171,9 +171,53 @@ void BitData::print_data_on(outputStream* st, const char* extra) const { // // A CounterData corresponds to a simple counter. -void CounterData::print_data_on(outputStream* st, const char* extra) const { +void CounterData::print_data_on(outputStream* st, const char* extra, bool cr) const { print_shared(st, "CounterData", extra); - st->print_cr("count(%u)", count()); + st->print("count(%u)", count()); + if (cr) st->cr(); +} + +// ================================================================== +// G1CounterData +// +// A G1CounterData corresponds to ... + +void G1CounterData::post_initialize(BytecodeStream* stream, MethodData* mdo) { + assert(stream->bci() == bci(), "wrong pos"); +} + +void G1CounterData::print_data_on(outputStream* st, const char* extra, bool cr) const { + print_shared(st, "G1CounterData ", extra); + st->print("Samples (%zu) Same-Region (%zu) Zero-Val (%zu) Clean-Cards (%zu) From-Young (%zu)", count(0), count(1), count(2), count(3), count(4)); + if (cr) st->cr(); +} + +// ================================================================== +// CombinedData +// +// A CombinedData corresponds to ... + +CombinedData::CombinedData(DataLayout* layout) : ProfileData(layout), _g1_counter(), _receiver_data() { + _g1_counter.set_data((DataLayout*)&layout->_cells[0]); + assert(_g1_counter.is_G1CounterData(), "must be"); + _receiver_data.set_data((DataLayout*)(&layout->_cells[0] + receiver_type_data_cell_offset())); + assert(_receiver_data.is_ReceiverTypeData(), "must be"); +} + +int CombinedData::receiver_type_data_cell_offset() { + return in_bytes(G1CounterData::counter_data_size()) / sizeof(intptr_t); +} + +void CombinedData::post_initialize(BytecodeStream* stream, MethodData* mdo) { + assert(stream->bci() == bci(), "wrong pos"); + _g1_counter.post_initialize(stream, mdo); + _receiver_data.post_initialize(stream, mdo); +} + +void CombinedData::print_data_on(outputStream* st, const char* extra, bool cr) const { + print_shared(st, "CombinedData ", extra); + _g1_counter.print_data_on(st, extra, cr); + _receiver_data.print_data_on(st, extra, cr); } // ================================================================== @@ -199,9 +243,10 @@ void JumpData::post_initialize(BytecodeStream* stream, MethodData* mdo) { set_displacement(offset); } -void JumpData::print_data_on(outputStream* st, const char* extra) const { +void JumpData::print_data_on(outputStream* st, const char* extra, bool cr) const { print_shared(st, "JumpData", extra); - st->print_cr("taken(%u) displacement(%d)", taken(), displacement()); + st->print("taken(%u) displacement(%d)", taken(), displacement()); + if (cr) st->cr(); } int TypeStackSlotEntries::compute_cell_count(Symbol* signature, bool include_receiver, int max) { @@ -358,46 +403,46 @@ void TypeEntries::print_klass(outputStream* st, intptr_t k) { } } -void TypeStackSlotEntries::print_data_on(outputStream* st) const { +void TypeStackSlotEntries::print_data_on(outputStream* st, bool cr) const { for (int i = 0; i < _number_of_entries; i++) { _pd->tab(st); st->print("%d: stack(%u) ", i, stack_slot(i)); print_klass(st, type(i)); - st->cr(); + if (cr) st->cr(); } } -void ReturnTypeEntry::print_data_on(outputStream* st) const { +void ReturnTypeEntry::print_data_on(outputStream* st, bool cr) const { _pd->tab(st); print_klass(st, type()); - st->cr(); + if (cr) st->cr(); } -void CallTypeData::print_data_on(outputStream* st, const char* extra) const { - CounterData::print_data_on(st, extra); +void CallTypeData::print_data_on(outputStream* st, const char* extra, bool cr) const { + CounterData::print_data_on(st, extra, cr); if (has_arguments()) { tab(st, true); st->print("argument types"); - _args.print_data_on(st); + _args.print_data_on(st, cr); } if (has_return()) { tab(st, true); st->print("return type"); - _ret.print_data_on(st); + _ret.print_data_on(st, cr); } } -void VirtualCallTypeData::print_data_on(outputStream* st, const char* extra) const { - VirtualCallData::print_data_on(st, extra); +void VirtualCallTypeData::print_data_on(outputStream* st, const char* extra, bool cr) const { + VirtualCallData::print_data_on(st, extra, cr); if (has_arguments()) { tab(st, true); st->print("argument types"); - _args.print_data_on(st); + _args.print_data_on(st, cr); } if (has_return()) { tab(st, true); st->print("return type"); - _ret.print_data_on(st); + _ret.print_data_on(st, cr); } } @@ -418,7 +463,7 @@ void ReceiverTypeData::clean_weak_klass_links(bool always_clean) { } } -void ReceiverTypeData::print_receiver_data_on(outputStream* st) const { +void ReceiverTypeData::print_receiver_data_on(outputStream* st, bool cr) const { uint row; int entries = 0; for (row = 0; row < row_limit(); row++) { @@ -435,18 +480,20 @@ void ReceiverTypeData::print_receiver_data_on(outputStream* st) const { if (receiver(row) != nullptr) { tab(st); receiver(row)->print_value_on(st); - st->print_cr("(%u %4.2f)", receiver_count(row), (float) receiver_count(row) / (float) total); + st->print("(%u %4.2f) ", receiver_count(row), (float) receiver_count(row) / (float) total); + if (cr) st->cr(); } } } -void ReceiverTypeData::print_data_on(outputStream* st, const char* extra) const { + +void ReceiverTypeData::print_data_on(outputStream* st, const char* extra, bool cr) const { print_shared(st, "ReceiverTypeData", extra); - print_receiver_data_on(st); + print_receiver_data_on(st, cr); } -void VirtualCallData::print_data_on(outputStream* st, const char* extra) const { +void VirtualCallData::print_data_on(outputStream* st, const char* extra, bool cr) const { print_shared(st, "VirtualCallData", extra); - print_receiver_data_on(st); + print_receiver_data_on(st, cr); } // ================================================================== @@ -493,19 +540,21 @@ address RetData::fixup_ret(int return_bci, MethodData* h_mdo) { return mdp; } -void RetData::print_data_on(outputStream* st, const char* extra) const { +void RetData::print_data_on(outputStream* st, const char* extra, bool cr) const { print_shared(st, "RetData", extra); uint row; int entries = 0; for (row = 0; row < row_limit(); row++) { if (bci(row) != no_bci) entries++; } - st->print_cr("count(%u) entries(%u)", count(), entries); + st->print("count(%u) entries(%u) ", count(), entries); + if (cr) st->cr(); for (row = 0; row < row_limit(); row++) { if (bci(row) != no_bci) { tab(st); - st->print_cr("bci(%d: count(%u) displacement(%d))", + st->print("bci(%d: count(%u) displacement(%d)) ", bci(row), bci_count(row), bci_displacement(row)); + if (cr) st->cr(); } } } @@ -526,12 +575,14 @@ void BranchData::post_initialize(BytecodeStream* stream, MethodData* mdo) { set_displacement(offset); } -void BranchData::print_data_on(outputStream* st, const char* extra) const { +void BranchData::print_data_on(outputStream* st, const char* extra, bool cr) const { print_shared(st, "BranchData", extra); - st->print_cr("taken(%u) displacement(%d)", - taken(), displacement()); + st->print("taken(%u) displacement(%d) ", + taken(), displacement()); + if (cr) st->cr(); tab(st); - st->print_cr("not taken(%u)", not_taken()); + st->print("not taken(%u)", not_taken()); + if (cr) st->cr(); } // ================================================================== @@ -598,25 +649,27 @@ void MultiBranchData::post_initialize(BytecodeStream* stream, } } -void MultiBranchData::print_data_on(outputStream* st, const char* extra) const { +void MultiBranchData::print_data_on(outputStream* st, const char* extra, bool cr) const { print_shared(st, "MultiBranchData", extra); - st->print_cr("default_count(%u) displacement(%d)", - default_count(), default_displacement()); + st->print("default_count(%u) displacement(%d) ", + default_count(), default_displacement()); + if (cr) st->cr(); int cases = number_of_cases(); for (int i = 0; i < cases; i++) { tab(st); - st->print_cr("count(%u) displacement(%d)", - count_at(i), displacement_at(i)); + st->print("count(%u) displacement(%d) ", + count_at(i), displacement_at(i)); + if (cr) st->cr(); } } -void ArgInfoData::print_data_on(outputStream* st, const char* extra) const { +void ArgInfoData::print_data_on(outputStream* st, const char* extra, bool cr) const { print_shared(st, "ArgInfoData", extra); int nargs = number_of_args(); for (int i = 0; i < nargs; i++) { st->print(" 0x%x", arg_modified(i)); } - st->cr(); + if (cr) st->cr(); } int ParametersTypeData::compute_cell_count(Method* m) { @@ -639,18 +692,18 @@ bool ParametersTypeData::profiling_enabled() { return MethodData::profile_parameters(); } -void ParametersTypeData::print_data_on(outputStream* st, const char* extra) const { +void ParametersTypeData::print_data_on(outputStream* st, const char* extra, bool cr) const { print_shared(st, "ParametersTypeData", extra); tab(st); _parameters.print_data_on(st); - st->cr(); + if (cr) st->cr(); } -void SpeculativeTrapData::print_data_on(outputStream* st, const char* extra) const { +void SpeculativeTrapData::print_data_on(outputStream* st, const char* extra, bool cr) const { print_shared(st, "SpeculativeTrapData", extra); tab(st); method()->print_short_name(st); - st->cr(); + if (cr) st->cr(); } // ================================================================== @@ -669,14 +722,27 @@ MethodData* MethodData::allocate(ClassLoaderData* loader_data, const methodHandl int MethodData::bytecode_cell_count(Bytecodes::Code code) { switch (code) { + case Bytecodes::_putstatic: + case Bytecodes::_putfield: + if (UseG1GC) { + return G1CounterData::static_cell_count(); + } else { + return no_profile_data; + } case Bytecodes::_checkcast: case Bytecodes::_instanceof: - case Bytecodes::_aastore: if (TypeProfileCasts) { return ReceiverTypeData::static_cell_count(); } else { return BitData::static_cell_count(); } + case Bytecodes::_aastore: + if (TypeProfileCasts) { + return UseNewCode ? CombinedData::static_cell_count() : ReceiverTypeData::static_cell_count(); + } else { + ShouldNotReachHere(); + return false; + } case Bytecodes::_invokespecial: case Bytecodes::_invokestatic: if (MethodData::profile_arguments() || MethodData::profile_return()) { @@ -993,9 +1059,16 @@ int MethodData::initialize_data(BytecodeStream* stream, DataLayout* data_layout = data_layout_at(data_index); Bytecodes::Code c = stream->code(); switch (c) { + case Bytecodes::_putstatic: + case Bytecodes::_putfield: { + if (UseG1GC) { + cell_count = G1CounterData::static_cell_count(); + tag = DataLayout::g1counter_data_tag; + } + break; + } case Bytecodes::_checkcast: case Bytecodes::_instanceof: - case Bytecodes::_aastore: if (TypeProfileCasts) { cell_count = ReceiverTypeData::static_cell_count(); tag = DataLayout::receiver_type_data_tag; @@ -1004,6 +1077,14 @@ int MethodData::initialize_data(BytecodeStream* stream, tag = DataLayout::bit_data_tag; } break; + case Bytecodes::_aastore: + if (TypeProfileCasts) { + cell_count = UseNewCode ? CombinedData::static_cell_count() : ReceiverTypeData::static_cell_count(); + tag = UseNewCode ? DataLayout::combined_data_tag : DataLayout::receiver_type_data_tag; + } else { + ShouldNotReachHere(); + } + break; case Bytecodes::_invokespecial: case Bytecodes::_invokestatic: { int counter_data_cell_count = CounterData::static_cell_count(); @@ -1100,7 +1181,15 @@ int MethodData::initialize_data(BytecodeStream* stream, if (cell_count >= 0) { assert(tag != DataLayout::no_tag, "bad tag"); assert(bytecode_has_profile(c), "agree w/ BHP"); - data_layout->initialize(tag, checked_cast(stream->bci()), cell_count); + u2 bci = checked_cast(stream->bci()); + data_layout->initialize(tag, bci, cell_count); + if (tag == DataLayout::combined_data_tag) { + DataLayout* temp; + temp = data_layout_at(data_index + DataLayout::header_size_in_cells() * sizeof(intptr_t /* type of cell */)); + temp->initialize(DataLayout::g1counter_data_tag, bci, G1CounterData::static_cell_count()); + temp = data_layout_at(data_index + DataLayout::header_size_in_cells() * sizeof (intptr_t) + in_bytes(G1CounterData::counter_data_size())); + temp->initialize(DataLayout::receiver_type_data_tag, bci, ReceiverTypeData::static_cell_count()); + } return DataLayout::compute_size_in_bytes(cell_count); } else { assert(!bytecode_has_profile(c), "agree w/ !BHP"); @@ -1127,6 +1216,10 @@ int DataLayout::cell_count() { return BitData::static_cell_count(); case DataLayout::counter_data_tag: return CounterData::static_cell_count(); + case DataLayout::g1counter_data_tag: + return G1CounterData::static_cell_count(); + case DataLayout::combined_data_tag: + return CombinedData::static_cell_count(); case DataLayout::jump_data_tag: return JumpData::static_cell_count(); case DataLayout::receiver_type_data_tag: @@ -1161,6 +1254,10 @@ ProfileData* DataLayout::data_in() { return new BitData(this); case DataLayout::counter_data_tag: return new CounterData(this); + case DataLayout::g1counter_data_tag: + return new G1CounterData(this); + case DataLayout::combined_data_tag: + return new CombinedData(this); case DataLayout::jump_data_tag: return new JumpData(this); case DataLayout::receiver_type_data_tag: @@ -1223,7 +1320,7 @@ void MethodData::post_initialize(BytecodeStream* stream) { MethodData::MethodData(const methodHandle& method) : _method(method()), // Holds Compile_lock - _extra_data_lock(Mutex::nosafepoint, "MDOExtraData_lock"), + _extra_data_lock(Mutex::nosafepoint-1, "MDOExtraData_lock"), _compiler_counters(), _parameters_type_data_di(parameters_uninitialized) { initialize(); diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index 8375552a911b2..fdbfbd00e2568 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -80,6 +80,7 @@ class ProfileData; class DataLayout { friend class VMStructs; friend class JVMCIVMStructs; + friend class CombinedData; private: // Every data layout begins with a header. This header @@ -119,6 +120,8 @@ class DataLayout { no_tag, bit_data_tag, counter_data_tag, + g1counter_data_tag, + combined_data_tag, jump_data_tag, receiver_type_data_tag, virtual_call_data_tag, @@ -288,6 +291,8 @@ class VirtualCallData; class VirtualCallTypeData; class RetData; class CallTypeData; +class CombinedData; +class G1CounterData; class JumpData; class BranchData; class ArrayData; @@ -304,6 +309,8 @@ class ProfileData : public ResourceObj { friend class TypeEntries; friend class ReturnTypeEntry; friend class TypeStackSlotEntries; + friend class CombinedData; + private: enum { tab_width_one = 16, @@ -343,7 +350,7 @@ class ProfileData : public ResourceObj { } void release_set_intptr_at(int index, intptr_t value); intptr_t intptr_at(int index) const { - assert(0 <= index && index < cell_count(), "oob"); + assert(0 <= index && index < cell_count(), "oob %d %d", index, cell_count()); return data()->cell_at(index); } void set_uint_at(int index, uint value) { @@ -386,6 +393,10 @@ class ProfileData : public ResourceObj { } ProfileData(DataLayout* data) { + set_data(data); + } + + void set_data(DataLayout* data) { _data = data; } @@ -411,6 +422,8 @@ class ProfileData : public ResourceObj { // Type checking virtual bool is_BitData() const { return false; } virtual bool is_CounterData() const { return false; } + virtual bool is_G1CounterData() const { return false; } + virtual bool is_CombinedData() const { return false; } virtual bool is_JumpData() const { return false; } virtual bool is_ReceiverTypeData()const { return false; } virtual bool is_VirtualCallData() const { return false; } @@ -433,11 +446,19 @@ class ProfileData : public ResourceObj { assert(is_CounterData(), "wrong type"); return is_CounterData() ? (CounterData*) this : nullptr; } + virtual G1CounterData* as_G1CounterData() const { + assert(is_G1CounterData(), "wrong type"); + return is_G1CounterData() ? (G1CounterData*) this : nullptr; + } + virtual CombinedData* as_CombinedData() const { + assert(is_CombinedData(), "wrong type"); + return is_CombinedData() ? (CombinedData*) this : nullptr; + } JumpData* as_JumpData() const { assert(is_JumpData(), "wrong type"); return is_JumpData() ? (JumpData*) this : nullptr; } - ReceiverTypeData* as_ReceiverTypeData() const { + virtual ReceiverTypeData* as_ReceiverTypeData() const { assert(is_ReceiverTypeData(), "wrong type"); return is_ReceiverTypeData() ? (ReceiverTypeData*)this : nullptr; } @@ -496,7 +517,7 @@ class ProfileData : public ResourceObj { // translation here, and the required translators are in the ci subclasses. virtual void translate_from(const ProfileData* data) {} - virtual void print_data_on(outputStream* st, const char* extra = nullptr) const { + virtual void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const { ShouldNotReachHere(); } @@ -526,8 +547,8 @@ class BitData : public ProfileData { }; enum { bit_cell_count = 0 }; // no additional data fields needed. public: - BitData(DataLayout* layout) : ProfileData(layout) { - } + BitData(DataLayout* layout) : ProfileData(layout) { } + BitData() : ProfileData() { } virtual bool is_BitData() const { return true; } @@ -544,7 +565,7 @@ class BitData : public ProfileData { // The null_seen flag bit is specially known to the interpreter. // Consulting it allows the compiler to avoid setting up null_check traps. bool null_seen() { return flag_at(null_seen_flag); } - void set_null_seen() { set_flag_at(null_seen_flag); } + //void set_null_seen() { set_flag_at(null_seen_flag); } bool deprecated_method_call_site() const { return flag_at(deprecated_method_callsite_flag); } bool set_deprecated_method_call_site() { return data()->set_flag_at(deprecated_method_callsite_flag); } bool clear_deprecated_method_call_site() { return data()->clear_flag_at(deprecated_method_callsite_flag); } @@ -568,7 +589,7 @@ class BitData : public ProfileData { return cell_offset(bit_cell_count); } - void print_data_on(outputStream* st, const char* extra = nullptr) const; + void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const; }; // CounterData @@ -584,6 +605,7 @@ class CounterData : public BitData { }; public: CounterData(DataLayout* layout) : BitData(layout) {} + CounterData() : BitData() {} virtual bool is_CounterData() const { return true; } @@ -618,7 +640,66 @@ class CounterData : public BitData { set_int_at(count_off, count); } - void print_data_on(outputStream* st, const char* extra = nullptr) const; + void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const; +}; + +// G1CounterData +// +// Four counters. +class G1CounterData : public ProfileData { + friend class VMStructs; + friend class JVMCIVMStructs; +protected: + enum { + count_off, + g1count_cell_count = 6 + }; + + intptr_t count(int idx) const { + intptr_t raw_data = intptr_at(count_off + idx); + return raw_data; + } + + enum CountIndexes { + Visits = 0, + SameRegion, + NullNewVal, + CleanCards, + FromYoung + }; + + static ByteSize count_offset(CountIndexes idx) { + return cell_offset(count_off + idx); + } + +public: + G1CounterData(DataLayout* layout) : ProfileData(layout) {} + G1CounterData() : ProfileData() {} + + virtual bool is_G1CounterData() const { return true; } + + static int static_cell_count() { return g1count_cell_count; } + int cell_count() const override { return G1CounterData::static_cell_count(); } + + static ByteSize visits_counter_offset() { return count_offset(Visits); } + static ByteSize same_region_counter_offset() { return count_offset(SameRegion); } + static ByteSize null_new_val_counter_offset() { return count_offset(NullNewVal); } + static ByteSize clean_cards_counter_offset() { return count_offset(CleanCards); } + static ByteSize from_young_counter_offset() { return count_offset(FromYoung); } + + intptr_t visits_count() const { return count(Visits); } + intptr_t same_region_count() const { return count(SameRegion); } + intptr_t null_new_val_count() const { return count(NullNewVal); } + intptr_t clean_cards_count() const { return count(CleanCards); } + intptr_t from_young_count() const { return count(FromYoung); } + + static ByteSize counter_data_size() { + return cell_offset(g1count_cell_count); + } + + void post_initialize(BytecodeStream* stream, MethodData* mdo) override; + + void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const override; }; // JumpData @@ -691,7 +772,7 @@ class JumpData : public ProfileData { // Specific initialization. void post_initialize(BytecodeStream* stream, MethodData* mdo); - void print_data_on(outputStream* st, const char* extra = nullptr) const; + void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const; }; // Entries in a ProfileData object to record types: it can either be @@ -875,7 +956,7 @@ class TypeStackSlotEntries : public TypeEntries { // GC support void clean_weak_klass_links(bool always_clean); - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, bool cr = true) const; }; // Type entry used for return from a call. A single cell to record the @@ -918,7 +999,7 @@ class ReturnTypeEntry : public TypeEntries { // GC support void clean_weak_klass_links(bool always_clean); - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, bool cr = true) const; }; // Entries to collect type information at a call: contains arguments @@ -1109,7 +1190,7 @@ class CallTypeData : public CounterData { } } - virtual void print_data_on(outputStream* st, const char* extra = nullptr) const; + virtual void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const; }; // ReceiverTypeData @@ -1135,9 +1216,11 @@ class ReceiverTypeData : public CounterData { ReceiverTypeData(DataLayout* layout) : CounterData(layout) { assert(layout->tag() == DataLayout::receiver_type_data_tag || layout->tag() == DataLayout::virtual_call_data_tag || - layout->tag() == DataLayout::virtual_call_type_data_tag, "wrong type"); + layout->tag() == DataLayout::virtual_call_type_data_tag, "wrong type %d", layout->tag()); } + ReceiverTypeData() : CounterData() {} + virtual bool is_ReceiverTypeData() const { return true; } static int static_cell_count() { @@ -1219,8 +1302,65 @@ class ReceiverTypeData : public CounterData { // GC support virtual void clean_weak_klass_links(bool always_clean); - void print_receiver_data_on(outputStream* st) const; - void print_data_on(outputStream* st, const char* extra = nullptr) const; + void print_receiver_data_on(outputStream* st, bool add_cr = true) const; + void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const; +}; + + +// CombinedData +// +// .... +class CombinedData : public ProfileData { + friend class VMStructs; + friend class JVMCIVMStructs; + + G1CounterData _g1_counter; + ReceiverTypeData _receiver_data; + +public: + CombinedData(DataLayout* layout); + + virtual bool is_CombinedData() const { return true; } + virtual bool is_G1CounterData() const { return _g1_counter.is_G1CounterData(); } + virtual bool is_ReceiverTypeData() const { return _receiver_data.is_ReceiverTypeData(); } + + virtual G1CounterData* as_G1CounterData() const { + assert(is_G1CounterData(), "wrong type"); + return (G1CounterData*)&_g1_counter; + } + virtual ReceiverTypeData* as_ReceiverTypeData() const { + assert(is_ReceiverTypeData(), "wrong type"); + return (ReceiverTypeData*)&_receiver_data; + } + + static int receiver_type_data_cell_offset(); + static int static_cell_count() { return 2 * DataLayout::header_size_in_cells() + G1CounterData::static_cell_count() + ReceiverTypeData::static_cell_count(); } + int cell_count() const override { return CombinedData::static_cell_count(); } + + /* + static ByteSize counter_data_size() { + return G1CounterData::counter_data_size() + ReceiverTypeData::receiver_type_data_size(); + }*/ + + static ByteSize g1_counter_data_offset() { + return in_ByteSize(DataLayout::header_size_in_bytes()); + } + + static ByteSize receiver_type_data_offset() { + return g1_counter_data_offset() + G1CounterData::counter_data_size(); + } + + static ByteSize receiver_type_data_size() { + return ReceiverTypeData::receiver_type_data_size(); + } + + static ByteSize combined_data_data_size() { + return in_ByteSize(DataLayout::header_size_in_bytes()) + G1CounterData::counter_data_size() + ReceiverTypeData::receiver_type_data_size(); + } + + void post_initialize(BytecodeStream* stream, MethodData* mdo) override; + + void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const override; }; // VirtualCallData @@ -1252,7 +1392,7 @@ class VirtualCallData : public ReceiverTypeData { } void print_method_data_on(outputStream* st) const NOT_JVMCI_RETURN; - void print_data_on(outputStream* st, const char* extra = nullptr) const; + void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const; }; // VirtualCallTypeData @@ -1384,7 +1524,7 @@ class VirtualCallTypeData : public VirtualCallData { } } - virtual void print_data_on(outputStream* st, const char* extra = nullptr) const; + virtual void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const; }; // RetData @@ -1476,7 +1616,7 @@ class RetData : public CounterData { // Specific initialization. void post_initialize(BytecodeStream* stream, MethodData* mdo); - void print_data_on(outputStream* st, const char* extra = nullptr) const; + void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const; }; // BranchData @@ -1540,7 +1680,7 @@ class BranchData : public JumpData { // Specific initialization. void post_initialize(BytecodeStream* stream, MethodData* mdo); - void print_data_on(outputStream* st, const char* extra = nullptr) const; + void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const; }; // ArrayData @@ -1699,7 +1839,7 @@ class MultiBranchData : public ArrayData { // Specific initialization. void post_initialize(BytecodeStream* stream, MethodData* mdo); - void print_data_on(outputStream* st, const char* extra = nullptr) const; + void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const; }; class ArgInfoData : public ArrayData { @@ -1724,7 +1864,7 @@ class ArgInfoData : public ArrayData { array_set_int_at(arg, val); } - void print_data_on(outputStream* st, const char* extra = nullptr) const; + void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const; }; // ParametersTypeData @@ -1783,7 +1923,7 @@ class ParametersTypeData : public ArrayData { _parameters.clean_weak_klass_links(always_clean); } - virtual void print_data_on(outputStream* st, const char* extra = nullptr) const; + virtual void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const; static ByteSize stack_slot_offset(int i) { return cell_offset(stack_slot_local_offset(i)); @@ -1853,7 +1993,7 @@ class SpeculativeTrapData : public ProfileData { return cell_offset(speculative_trap_method); } - virtual void print_data_on(outputStream* st, const char* extra = nullptr) const; + virtual void print_data_on(outputStream* st, const char* extra = nullptr, bool add_cr = true) const; }; // MethodData* diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index 0cc68c426398e..db554999032d4 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -1584,7 +1584,8 @@ Node* GraphKit::store_to_memory(Node* ctl, Node* adr, Node *val, BasicType bt, bool unaligned, bool mismatched, bool unsafe, - int barrier_data) { + int barrier_data, + int ext_barrier_data) { int adr_idx = C->get_alias_index(_gvn.type(adr)->isa_ptr()); assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" ); const TypePtr* adr_type = nullptr; @@ -1601,6 +1602,7 @@ Node* GraphKit::store_to_memory(Node* ctl, Node* adr, Node *val, BasicType bt, st->as_Store()->set_unsafe_access(); } st->as_Store()->set_barrier_data(barrier_data); + st->as_Store()->set_ext_barrier_data(ext_barrier_data); st = _gvn.transform(st); set_memory(st, adr_idx); // Back-to-back stores can only remove intermediate store with DU info @@ -2281,7 +2283,28 @@ Node* GraphKit::record_profiled_receiver_for_speculation(Node* n) { method()->method_data()->is_mature()) { ciProfileData* data = method()->method_data()->bci_to_data(bci()); if (data != nullptr) { - if (!data->as_BitData()->null_seen()) { + // The next code that checks whether there is any non-NULL receiver is completely + // broken due to how the ci* and the corresponding non-ci* typedata hierarchy is + // done. + // I.e. Combineddata contains ReceiverDataType and Klass* references in the + // receiverdata, and the receivertypedata stuff contains ci* klasses. + // FIXME. + if (data->is_CombinedData()) { + ReceiverTypeData* call = data->as_ReceiverTypeData(); + if (!call->as_BitData()->null_seen()) { + ptr_kind = ProfileNeverNull; + } else { + uint i = 0; + for (; i < call->row_limit(); i++) { + Klass* receiver = call->receiver(i); + if (receiver != nullptr) { + break; + } + } + ptr_kind = (i == call->row_limit()) ? ProfileAlwaysNull : ProfileMaybeNull; + } + + } else if (!data->as_BitData()->null_seen()) { ptr_kind = ProfileNeverNull; } else { assert(data->is_ReceiverTypeData(), "bad profile data type"); @@ -3313,6 +3336,9 @@ Node* GraphKit::gen_checkcast(Node *obj, Node* superklass, java_bc() == Bytecodes::_checkcast, "interpreter profiles type checks only for these BCs"); data = method()->method_data()->bci_to_data(bci()); + if (java_bc() == Bytecodes::_aastore && data != nullptr) { // FIXME: remove; check that we were going the right path. + data = data->as_ReceiverTypeData(); + } safe_for_replace = true; } diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp index 223f2ce83f544..cf51395bb7a79 100644 --- a/src/hotspot/share/opto/graphKit.hpp +++ b/src/hotspot/share/opto/graphKit.hpp @@ -559,7 +559,8 @@ class GraphKit : public Phase { bool unaligned = false, bool mismatched = false, bool unsafe = false, - int barrier_data = 0); + int barrier_data = 0, + int ext_barrier_data = 0); // Perform decorated accesses diff --git a/src/hotspot/share/opto/lcm.cpp b/src/hotspot/share/opto/lcm.cpp index 0079f366640e8..4a2ba4f1e83c3 100644 --- a/src/hotspot/share/opto/lcm.cpp +++ b/src/hotspot/share/opto/lcm.cpp @@ -161,7 +161,7 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo Node *m = val->out(i); if( !m->is_Mach() ) continue; MachNode *mach = m->as_Mach(); - if (mach->barrier_data() != 0) { + if (mach->barrier_data() != 0) { // FIXME // Using memory accesses with barriers to perform implicit null checks is // not supported. These operations might expand into multiple assembly // instructions during code emission, including new memory accesses (e.g. diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index a33d89993addd..d9f1b7fca88a1 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -1011,7 +1011,7 @@ bool IdealLoopTree::policy_unroll(PhaseIdealLoop *phase) { // Also count ModL, DivL, MulL, and other nodes that expand mightly for (uint k = 0; k < _body.size(); k++) { Node* n = _body.at(k); - if (MemNode::barrier_data(n) != 0) { + if (MemNode::barrier_data(n) != 0) { // FIXME body_size += BarrierSet::barrier_set()->barrier_set_c2()->estimated_barrier_size(n); } switch (n->Opcode()) { diff --git a/src/hotspot/share/opto/machnode.cpp b/src/hotspot/share/opto/machnode.cpp index e271637893b28..924d728863d19 100644 --- a/src/hotspot/share/opto/machnode.cpp +++ b/src/hotspot/share/opto/machnode.cpp @@ -551,7 +551,7 @@ void MachNode::dump_spec(outputStream *st) const { if( C->alias_type(t)->is_volatile() ) st->print(" Volatile!"); } - if (barrier_data() != 0) { + if (barrier_data() != 0) { // FIXME st->print(" barrier("); BarrierSet::barrier_set()->barrier_set_c2()->dump_barrier_data(this, st); st->print(") "); diff --git a/src/hotspot/share/opto/machnode.hpp b/src/hotspot/share/opto/machnode.hpp index 4ac91175f78ed..ad7eceb93a6bb 100644 --- a/src/hotspot/share/opto/machnode.hpp +++ b/src/hotspot/share/opto/machnode.hpp @@ -219,7 +219,7 @@ class MachOper : public ResourceObj { // ADLC inherit from this class. class MachNode : public Node { public: - MachNode() : Node((uint)0), _barrier(0), _num_opnds(0), _opnds(nullptr) { + MachNode() : Node((uint)0), _barrier(0), _ext_barrier(0), _num_opnds(0), _opnds(nullptr) { init_class_id(Class_Mach); } // Required boilerplate @@ -236,6 +236,9 @@ class MachNode : public Node { uint8_t barrier_data() const { return _barrier; } void set_barrier_data(uint8_t data) { _barrier = data; } + uint8_t ext_barrier_data() const { return _ext_barrier; } + void set_ext_barrier_data(uint8_t data) { _ext_barrier = data; } + // Copy index, inputs, and operands to a new version of the instruction. // Called from cisc_version() and short_branch_version(). void fill_new_machnode(MachNode *n) const; @@ -283,6 +286,8 @@ class MachNode : public Node { // The GC might require some barrier metadata for machine code emission. uint8_t _barrier; + uint8_t _ext_barrier; + // Array of complex operand pointers. Each corresponds to zero or // more leafs. Must be set by MachNode constructor to point to an // internal array of MachOpers. The MachOper array is sized by diff --git a/src/hotspot/share/opto/matcher.cpp b/src/hotspot/share/opto/matcher.cpp index b94fb7101430e..7b9038d75d1e8 100644 --- a/src/hotspot/share/opto/matcher.cpp +++ b/src/hotspot/share/opto/matcher.cpp @@ -1930,6 +1930,7 @@ MachNode *Matcher::ReduceInst( State *s, int rule, Node *&mem ) { // Have mach nodes inherit GC barrier data mach->set_barrier_data(MemNode::barrier_data(leaf)); + mach->set_ext_barrier_data(MemNode::ext_barrier_data(leaf)); return ex; } @@ -2872,7 +2873,7 @@ bool Matcher::is_encode_and_store_pattern(const Node* n, const Node* m) { m == nullptr || n->Opcode() != Op_StoreN || !m->is_EncodeP() || - n->as_Store()->barrier_data() == 0) { + n->as_Store()->barrier_data() == 0) { // suspicious return false; } assert(m == n->in(MemNode::ValueIn), "m should be input to n"); diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp index ad351fb81add7..5f5f9f9b91f41 100644 --- a/src/hotspot/share/opto/memnode.cpp +++ b/src/hotspot/share/opto/memnode.cpp @@ -870,6 +870,15 @@ uint8_t MemNode::barrier_data(const Node* n) { return 0; } +uint8_t MemNode::ext_barrier_data(const Node* n) { + if (n->is_LoadStore()) { + return n->as_LoadStore()->ext_barrier_data(); + } else if (n->is_Mem()) { + return n->as_Mem()->ext_barrier_data(); + } + return 0; +} + //============================================================================= // Should LoadNode::Ideal() attempt to remove control edges? bool LoadNode::can_remove_control() const { @@ -3647,7 +3656,7 @@ LoadStoreNode::LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const Ty : Node(required), _type(rt), _adr_type(at), - _barrier_data(0) + _barrier_data(0), _ext_barrier_data(0) { init_req(MemNode::Control, c ); init_req(MemNode::Memory , mem); diff --git a/src/hotspot/share/opto/memnode.hpp b/src/hotspot/share/opto/memnode.hpp index ee9db7d1cac1c..d365c032da9cb 100644 --- a/src/hotspot/share/opto/memnode.hpp +++ b/src/hotspot/share/opto/memnode.hpp @@ -45,6 +45,7 @@ class MemNode : public Node { bool _mismatched_access; // Mismatched access from unsafe: byte read in integer array for instance bool _unsafe_access; // Access of unsafe origin. uint8_t _barrier_data; // Bit field with barrier information + uint8_t _ext_barrier_data; protected: #ifdef ASSERT @@ -69,7 +70,7 @@ class MemNode : public Node { _unaligned_access(false), _mismatched_access(false), _unsafe_access(false), - _barrier_data(0) { + _barrier_data(0), _ext_barrier_data(0) { init_class_id(Class_Mem); debug_only(_adr_type=at; adr_type();) } @@ -78,7 +79,7 @@ class MemNode : public Node { _unaligned_access(false), _mismatched_access(false), _unsafe_access(false), - _barrier_data(0) { + _barrier_data(0), _ext_barrier_data(0) { init_class_id(Class_Mem); debug_only(_adr_type=at; adr_type();) } @@ -87,7 +88,7 @@ class MemNode : public Node { _unaligned_access(false), _mismatched_access(false), _unsafe_access(false), - _barrier_data(0) { + _barrier_data(0), _ext_barrier_data(0) { init_class_id(Class_Mem); debug_only(_adr_type=at; adr_type();) } @@ -128,6 +129,7 @@ class MemNode : public Node { // Return the barrier data of n, if available, or 0 otherwise. static uint8_t barrier_data(const Node* n); + static uint8_t ext_barrier_data(const Node* n); // Map a load or store opcode to its corresponding store opcode. // (Return -1 if unknown.) @@ -145,6 +147,8 @@ class MemNode : public Node { uint8_t barrier_data() { return _barrier_data; } void set_barrier_data(uint8_t barrier_data) { _barrier_data = barrier_data; } + uint8_t ext_barrier_data() { return _ext_barrier_data; } + void set_ext_barrier_data(uint8_t barrier_data) { _ext_barrier_data = barrier_data; } // Search through memory states which precede this node (load or store). // Look for an exact match for the address, with no intervening @@ -811,6 +815,7 @@ class LoadStoreNode : public Node { const Type* const _type; // What kind of value is loaded? const TypePtr* _adr_type; // What kind of memory is being addressed? uint8_t _barrier_data; // Bit field with barrier information + uint8_t _ext_barrier_data; virtual uint size_of() const; // Size is bigger public: LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* rt, uint required ); @@ -827,6 +832,9 @@ class LoadStoreNode : public Node { uint8_t barrier_data() { return _barrier_data; } void set_barrier_data(uint8_t barrier_data) { _barrier_data = barrier_data; } + + uint8_t ext_barrier_data() { return _ext_barrier_data; } + void set_ext_barrier_data(uint8_t barrier_data) { _ext_barrier_data = barrier_data; } }; class LoadStoreConditionalNode : public LoadStoreNode { diff --git a/src/hotspot/share/runtime/mutexLocker.cpp b/src/hotspot/share/runtime/mutexLocker.cpp index 3d5b49f56ce5a..e5f29acfb374b 100644 --- a/src/hotspot/share/runtime/mutexLocker.cpp +++ b/src/hotspot/share/runtime/mutexLocker.cpp @@ -76,6 +76,7 @@ Mutex* G1RareEvent_lock = nullptr; Mutex* G1DetachedRefinementStats_lock = nullptr; Mutex* MarkStackFreeList_lock = nullptr; Mutex* MarkStackChunkList_lock = nullptr; +Mutex* ByteCodeTracer_lock = nullptr; Mutex* MonitoringSupport_lock = nullptr; Monitor* ConcurrentGCBreakpoints_lock = nullptr; Mutex* Compile_lock = nullptr; @@ -214,6 +215,8 @@ void mutex_init() { MUTEX_DEFN(MarkStackFreeList_lock , PaddedMutex , nosafepoint); MUTEX_DEFN(MarkStackChunkList_lock , PaddedMutex , nosafepoint); } + MUTEX_DEFN(ByteCodeTracer_lock , PaddedMonitor, nosafepoint); + MUTEX_DEFN(MonitoringSupport_lock , PaddedMutex , service-1); // used for serviceability monitoring support MUTEX_DEFN(StringDedup_lock , PaddedMonitor, nosafepoint); diff --git a/src/hotspot/share/runtime/mutexLocker.hpp b/src/hotspot/share/runtime/mutexLocker.hpp index 0e0c043544363..73899687ff913 100644 --- a/src/hotspot/share/runtime/mutexLocker.hpp +++ b/src/hotspot/share/runtime/mutexLocker.hpp @@ -52,6 +52,7 @@ extern Monitor* Heap_lock; // a lock on the heap #ifdef INCLUDE_PARALLELGC extern Mutex* PSOldGenExpand_lock; // a lock on expanding the heap #endif +extern Mutex* ByteCodeTracer_lock; extern Mutex* AdapterHandlerLibrary_lock; // a lock on the AdapterHandlerLibrary extern Mutex* SignatureHandlerLibrary_lock; // a lock on the SignatureHandlerLibrary extern Mutex* VtableStubs_lock; // a lock on the VtableStubs diff --git a/src/hotspot/share/runtime/vmStructs.cpp b/src/hotspot/share/runtime/vmStructs.cpp index eaf259cedbaed..1bbe03b9ef9eb 100644 --- a/src/hotspot/share/runtime/vmStructs.cpp +++ b/src/hotspot/share/runtime/vmStructs.cpp @@ -2137,6 +2137,7 @@ declare_constant(DataLayout::no_tag) \ declare_constant(DataLayout::bit_data_tag) \ declare_constant(DataLayout::counter_data_tag) \ + declare_constant(DataLayout::g1counter_data_tag) \ declare_constant(DataLayout::jump_data_tag) \ declare_constant(DataLayout::receiver_type_data_tag) \ declare_constant(DataLayout::virtual_call_data_tag) \ diff --git a/src/hotspot/share/utilities/globalDefinitions.hpp b/src/hotspot/share/utilities/globalDefinitions.hpp index ccd3106b471a2..1306280bc02b7 100644 --- a/src/hotspot/share/utilities/globalDefinitions.hpp +++ b/src/hotspot/share/utilities/globalDefinitions.hpp @@ -1139,6 +1139,10 @@ inline T clamp(T value, T min, T max) { return MIN2(MAX2(value, min), max); } +inline double clamp_unit(double value) { + return clamp(value, 0.0, 1.0); +} + inline bool is_odd (intx x) { return x & 1; } inline bool is_even(intx x) { return !is_odd(x); } diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/DataLayout.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/DataLayout.java index d627c66839bd4..1e4f28cd2298e 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/DataLayout.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/DataLayout.java @@ -35,17 +35,18 @@ public class DataLayout { public static final int noTag = 0; public static final int bitDataTag = 1; public static final int counterDataTag = 2; - public static final int jumpDataTag= 3; - public static final int receiverTypeDataTag = 4; - public static final int virtualCallDataTag = 5; - public static final int retDataTag = 6; - public static final int branchDataTag = 7; - public static final int multiBranchDataTag = 8; - public static final int argInfoDataTag = 9; - public static final int callTypeDataTag = 10; - public static final int virtualCallTypeDataTag = 11; - public static final int parametersTypeDataTag = 12; - public static final int speculativeTrapDataTag = 13; + public static final int g1CounterDataTag = 3; + public static final int jumpDataTag= 4; + public static final int receiverTypeDataTag = 5; + public static final int virtualCallDataTag = 6; + public static final int retDataTag = 7; + public static final int branchDataTag = 8; + public static final int multiBranchDataTag = 9; + public static final int argInfoDataTag = 10; + public static final int callTypeDataTag = 11; + public static final int virtualCallTypeDataTag = 12; + public static final int parametersTypeDataTag = 13; + public static final int speculativeTrapDataTag = 14; // The trap state breaks down as [recompile:1 | reason:31]. // This further breakdown is defined in deoptimization.cpp. diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/G1CounterData.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/G1CounterData.java new file mode 100644 index 0000000000000..adb96faafc732 --- /dev/null +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/G1CounterData.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.oops; + +import java.io.*; +import java.util.*; +import sun.jvm.hotspot.debugger.*; +import sun.jvm.hotspot.runtime.*; +import sun.jvm.hotspot.types.*; +import sun.jvm.hotspot.utilities.*; + +// CounterData +// +// A CounterData corresponds to a simple counter. +public class G1CounterData extends ProfileData { + + static final int countOff = 0; + static final int counterCellCount = 4; + + public G1CounterData(DataLayout layout) { + super(layout); + } + + static int staticCellCount() { + return counterCellCount; + } + + public int cellCount() { + return staticCellCount(); + } + + // Direct accessor + int count(int index) { + return uintAt(index); + } + + // Code generation support + static int countOffset(int index) { + return cellOffset(index); + } + static int counterDataSize() { + return cellOffset(counterCellCount); + } + + public void printDataOn(PrintStream st) { + printShared(st, "CounterData"); + st.println("count0(" + count(0) + ")"); + st.println("count1(" + count(1) + ")"); + st.println("count2(" + count(2) + ")"); + st.println("count3(" + count(3) + ")"); + } +} diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/MethodData.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/MethodData.java index 0c05cabe32d51..381fa9713d461 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/MethodData.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/MethodData.java @@ -284,6 +284,8 @@ ProfileData dataAt(int dataIndex) { return new BitData(dataLayout); case DataLayout.counterDataTag: return new CounterData(dataLayout); + case DataLayout.g1CounterDataTag: + return new G1CounterData(dataLayout); case DataLayout.jumpDataTag: return new JumpData(dataLayout); case DataLayout.receiverTypeDataTag: diff --git a/test/micro/org/openjdk/bench/vm/compiler/WriteBarrier.java b/test/micro/org/openjdk/bench/vm/compiler/WriteBarrier.java index 78be9bcdaba5c..599b84169f0b0 100644 --- a/test/micro/org/openjdk/bench/vm/compiler/WriteBarrier.java +++ b/test/micro/org/openjdk/bench/vm/compiler/WriteBarrier.java @@ -167,6 +167,7 @@ public void testArrayWriteBarrierFastPathNullLarge() { } @Benchmark() + @CompilerControl(CompilerControl.Mode.DONT_INLINE) public void testFieldWriteBarrierFastPath() { // Shuffle everything around this.tail.append(this.head);