Skip to content

Commit 068a1d5

Browse files
author
Thomas Schatzl
committed
Optimizatins
1 parent 450b27d commit 068a1d5

File tree

8 files changed

+149
-45
lines changed

8 files changed

+149
-45
lines changed

src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -223,14 +223,14 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
223223
// get the address of the card
224224
__ load_byte_map_base(tmp2);
225225
__ add(card_addr, card_addr, tmp2);
226-
__ ldrb(tmp2, Address(card_addr));
227-
__ cmpw(tmp2, (int)G1CardTable::g1_young_card_val());
228-
__ br(Assembler::EQ, done);
226+
if (!UseNewCode) {
227+
__ ldrb(tmp2, Address(card_addr));
228+
__ cmpw(tmp2, (int)G1CardTable::g1_young_card_val());
229+
__ br(Assembler::EQ, done);
229230

231+
__ membar(Assembler::StoreLoad);
232+
}
230233
assert((int)CardTable::dirty_card_val() == 0, "must be 0");
231-
232-
__ membar(Assembler::StoreLoad);
233-
234234
__ ldrb(tmp2, Address(card_addr));
235235
__ cbzw(tmp2, done);
236236

@@ -444,13 +444,14 @@ void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler*
444444
__ load_parameter(0, card_offset);
445445
__ lsr(card_offset, card_offset, CardTable::card_shift());
446446
__ load_byte_map_base(byte_map_base);
447-
__ ldrb(rscratch1, Address(byte_map_base, card_offset));
448-
__ cmpw(rscratch1, (int)G1CardTable::g1_young_card_val());
449-
__ br(Assembler::EQ, done);
447+
if (!UseNewCode) {
448+
__ ldrb(rscratch1, Address(byte_map_base, card_offset));
449+
__ cmpw(rscratch1, (int)G1CardTable::g1_young_card_val());
450+
__ br(Assembler::EQ, done);
450451

452+
__ membar(Assembler::StoreLoad);
453+
}
451454
assert((int)CardTable::dirty_card_val() == 0, "must be 0");
452-
453-
__ membar(Assembler::StoreLoad);
454455
__ ldrb(rscratch1, Address(byte_map_base, card_offset));
455456
__ cbzw(rscratch1, done);
456457

src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp

Lines changed: 84 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,80 @@ void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm
9898

9999
void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
100100
Register addr, Register count, Register tmp) {
101+
102+
if (true) {
103+
assert(sizeof(CardTable::CardValue) == 1, "must be");
104+
105+
Label done;
106+
__ testptr(count, count);
107+
__ jcc(Assembler::equal, done); // nothing to do if empty ref array.
108+
109+
// Calculate end address.
110+
__ shlptr(count, LogBytesPerHeapOop);
111+
__ addptr(count, addr);
112+
// Calculate start card address in "addr".
113+
__ shrptr(addr, CardTable::card_shift());
114+
__ movptr(tmp, (intptr_t)barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set())->card_table()->byte_map_base());
115+
__ addptr(addr, tmp);
116+
117+
if (!UseNewCode) {
118+
// If the object starts in a young region, there is nothing to do.
119+
__ cmpb(Address(addr, 0), G1CardTable::g1_young_card_val());
120+
__ jcc(Assembler::equal, done);
121+
__ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
122+
}
123+
124+
// Caclulate card address of last word.
125+
__ subptr(count, 1);
126+
__ shrptr(count, CardTable::card_shift());
127+
__ addptr(count, tmp);
128+
129+
Label loop;
130+
__ bind(loop);
131+
132+
Label next_card;
133+
__ cmpb(Address(addr, 0), G1CardTable::dirty_card_val());
134+
__ jcc(Assembler::zero, next_card);
135+
136+
// Card was not dirty. Dirty card and enqueue.
137+
__ movb(Address(addr, 0), G1CardTable::dirty_card_val());
138+
139+
Address queue_index(r15_thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
140+
Address buffer(r15_thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
141+
142+
__ movptr(tmp, queue_index);
143+
__ testptr(tmp, tmp);
144+
Label runtime;
145+
__ jcc(Assembler::zero, runtime);
146+
__ subptr(tmp, wordSize);
147+
__ movptr(queue_index, tmp);
148+
__ addptr(tmp, buffer);
149+
__ movptr(Address(tmp, 0), addr);
150+
__ jmp(next_card);
151+
152+
__ bind(runtime);
153+
154+
// Save caller saved registers.
155+
__ push_call_clobbered_registers(false /* save_fpu */);
156+
// FIXME: probably issue with Windows....
157+
if (c_rarg1 != r15_thread) {
158+
__ mov(c_rarg1, r15_thread);
159+
}
160+
if (c_rarg0 != addr) {
161+
__ mov(c_rarg0, addr);
162+
}
163+
164+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), c_rarg0, c_rarg1);
165+
__ pop_call_clobbered_registers(false /* save_fpu */);
166+
167+
__ bind(next_card);
168+
__ addptr(addr, sizeof(CardTable::CardValue));
169+
__ cmpptr(addr, count);
170+
__ jcc(Assembler::belowEqual, loop);
171+
172+
__ bind(done);
173+
return;
174+
}
101175
__ push_call_clobbered_registers(false /* save_fpu */);
102176
#ifdef _LP64
103177
if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
@@ -308,10 +382,12 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
308382
__ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
309383
__ addptr(card_addr, cardtable);
310384

311-
__ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
312-
__ jcc(Assembler::equal, done);
385+
if (!UseNewCode) {
386+
__ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
387+
__ jcc(Assembler::equal, done);
313388

314-
__ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
389+
__ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
390+
}
315391
__ cmpb(Address(card_addr, 0), G1CardTable::dirty_card_val());
316392
__ jcc(Assembler::equal, done);
317393

@@ -541,11 +617,12 @@ void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler*
541617
__ addptr(card_addr, cardtable);
542618

543619
NOT_LP64(__ get_thread(thread);)
620+
if (!UseNewCode) {
621+
__ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
622+
__ jcc(Assembler::equal, done);
544623

545-
__ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
546-
__ jcc(Assembler::equal, done);
547-
548-
__ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
624+
__ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
625+
}
549626
__ cmpb(Address(card_addr, 0), CardTable::dirty_card_val());
550627
__ jcc(Assembler::equal, done);
551628

src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -465,20 +465,25 @@ void G1BarrierSetC2::post_barrier(GraphKit* kit,
465465
// Ok must mark the card if not already dirty
466466

467467
// load the original value of the card
468-
Node* card_val = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw);
469468

469+
if (!UseNewCode) {
470+
Node* card_val = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw);
470471
__ if_then(card_val, BoolTest::ne, young_card, unlikely); {
471-
if (!UseNewCode) {
472-
kit->sync_kit(ideal);
473-
kit->insert_mem_bar(Op_MemBarVolatile, oop_store);
474-
__ sync_kit(kit);
475-
}
472+
kit->sync_kit(ideal);
473+
kit->insert_mem_bar(Op_MemBarVolatile, oop_store);
474+
__ sync_kit(kit);
476475

477476
Node* card_val_reload = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw);
478477
__ if_then(card_val_reload, BoolTest::ne, dirty_card); {
479478
g1_mark_card(kit, ideal, card_adr, oop_store, alias_idx, index, index_adr, buffer, tf);
480479
} __ end_if();
481480
} __ end_if();
481+
} else {
482+
Node* card_val_reload = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw);
483+
__ if_then(card_val_reload, BoolTest::ne, dirty_card); {
484+
g1_mark_card(kit, ideal, card_adr, oop_store, alias_idx, index, index_adr, buffer, tf);
485+
} __ end_if();
486+
}
482487
} __ end_if();
483488
} __ end_if();
484489
} else {
@@ -488,7 +493,7 @@ void G1BarrierSetC2::post_barrier(GraphKit* kit,
488493
// are set to 'g1_young_gen' (see G1CardTable::verify_g1_young_region()).
489494
assert(!use_ReduceInitialCardMarks(), "can only happen with card marking");
490495
Node* card_val = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw);
491-
__ if_then(card_val, BoolTest::ne, young_card); {
496+
__ if_then(card_val, BoolTest::ne, !UseNewCode ? young_card : dirty_card); {
492497
g1_mark_card(kit, ideal, card_adr, oop_store, alias_idx, index, index_adr, buffer, tf);
493498
} __ end_if();
494499
}

src/hotspot/share/gc/g1/g1BarrierSet.cpp

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,9 @@ void G1BarrierSet::write_ref_array_pre(narrowOop* dst, size_t count, bool dest_u
9393
void G1BarrierSet::write_ref_field_post_slow(volatile CardValue* byte) {
9494
// In the slow path, we know a card is not young
9595
assert(*byte != G1CardTable::g1_young_card_val(), "slow path invoked without filtering");
96-
OrderAccess::storeload();
96+
if (!UseNewCode) {
97+
OrderAccess::storeload();
98+
}
9799
if (*byte != G1CardTable::dirty_card_val()) {
98100
*byte = G1CardTable::dirty_card_val();
99101
Thread* thr = Thread::current();
@@ -110,22 +112,30 @@ void G1BarrierSet::invalidate(JavaThread* thread, MemRegion mr) {
110112
CardValue* last_byte = _card_table->byte_for(mr.last());
111113

112114
// skip young gen cards
113-
if (*byte == G1CardTable::g1_young_card_val()) {
114-
// MemRegion should not span multiple regions for the young gen.
115-
DEBUG_ONLY(HeapRegion* containing_hr = G1CollectedHeap::heap()->heap_region_containing(mr.start());)
116-
assert(containing_hr->is_young(), "it should be young");
117-
assert(containing_hr->is_in(mr.start()), "it should contain start");
118-
assert(containing_hr->is_in(mr.last()), "it should also contain last");
119-
return;
115+
if (!UseNewCode) {
116+
if (*byte == G1CardTable::g1_young_card_val()) {
117+
// MemRegion should not span multiple regions for the young gen.
118+
DEBUG_ONLY(HeapRegion* containing_hr = G1CollectedHeap::heap()->heap_region_containing(mr.start());)
119+
assert(containing_hr->is_young(), "it should be young");
120+
assert(containing_hr->is_in(mr.start()), "it should contain start");
121+
assert(containing_hr->is_in(mr.last()), "it should also contain last");
122+
return;
123+
}
124+
OrderAccess::storeload();
125+
} else {
126+
HeapRegion* containing_hr = G1CollectedHeap::heap()->heap_region_containing(mr.start());
127+
if (containing_hr->is_young()) {
128+
assert(containing_hr->is_in(mr.last()), "it should also contain last");
129+
return;
130+
}
120131
}
121132

122-
OrderAccess::storeload();
123133
// Enqueue if necessary.
124134
G1DirtyCardQueueSet& qset = G1BarrierSet::dirty_card_queue_set();
125135
G1DirtyCardQueue& queue = G1ThreadLocalData::dirty_card_queue(thread);
126136
for (; byte <= last_byte; byte++) {
127137
CardValue bv = *byte;
128-
assert(bv != G1CardTable::g1_young_card_val(), "Invalid card");
138+
assert(UseNewCode || bv != G1CardTable::g1_young_card_val(), "Invalid card");
129139
if (bv != G1CardTable::dirty_card_val()) {
130140
*byte = G1CardTable::dirty_card_val();
131141
qset.enqueue(queue, byte);

src/hotspot/share/gc/g1/g1BarrierSet.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
* 2 along with this work; if not, write to the Free Software Foundation,
1717
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
1818
*
19-
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19+
* Please contact Oracle, 500 Oracle Parkway, Redwowrite_ref_arrayod Shores, CA 94065 USA
2020
* or visit www.oracle.com if you need additional information or have any
2121
* questions.
2222
*

src/hotspot/share/gc/g1/g1BarrierSet.inline.hpp

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,10 +83,13 @@ inline void G1BarrierSet::write_ref_array_work(MemRegion mr) {
8383
template <DecoratorSet decorators, typename T>
8484
inline void G1BarrierSet::write_ref_field_post(T* field) {
8585
volatile CardValue* byte = _card_table->byte_for(field);
86-
if (*byte != G1CardTable::g1_young_card_val()) {
87-
// Take a slow path for cards in old
88-
write_ref_field_post_slow(byte);
86+
if (!UseNewCode) {
87+
if (*byte == G1CardTable::g1_young_card_val()) {
88+
return;
89+
}
8990
}
91+
// Take a slow path for cards in old
92+
write_ref_field_post_slow(byte);
9093
}
9194

9295
inline void G1BarrierSet::enqueue_preloaded_if_weak(DecoratorSet decorators, oop value) {

src/hotspot/share/gc/g1/g1BarrierSetRuntime.cpp

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,16 @@ void G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry(narrowOop* dst, s
3939
bs->write_ref_array_pre(dst, length, false);
4040
}
4141

42-
void G1BarrierSetRuntime::write_ref_array_post_entry(HeapWord* dst, size_t length) {
42+
void G1BarrierSetRuntime::write_ref_array_post_entry(HeapWord* start, size_t length) {
43+
HeapWord* end = (HeapWord*)((char*)start + (length*heapOopSize));
44+
45+
HeapWord* aligned_start = align_down(start, HeapWordSize);
46+
HeapWord* aligned_end = align_up (end, HeapWordSize);
47+
// If compressed oops were not being used, these should already be aligned
48+
assert(UseCompressedOops || (aligned_start == start && aligned_end == end),
49+
"Expected heap word alignment of start and end");
4350
G1BarrierSet *bs = barrier_set_cast<G1BarrierSet>(BarrierSet::barrier_set());
44-
bs->G1BarrierSet::write_ref_array(dst, length);
51+
bs->write_ref_array_work(MemRegion(aligned_start, aligned_end));
4552
}
4653

4754
// G1 pre write barrier slowpath

src/hotspot/share/gc/g1/g1CardTable.cpp

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,12 @@ void G1CardTable::g1_mark_as_young(const MemRegion& mr) {
3232
CardValue *const first = byte_for(mr.start());
3333
CardValue *const last = byte_after(mr.last());
3434

35-
memset_with_concurrent_readers(first, g1_young_gen, pointer_delta(last, first, sizeof(CardValue)));
35+
memset_with_concurrent_readers(first, !UseNewCode ? g1_young_card_val() : dirty_card_val(), pointer_delta(last, first, sizeof(CardValue)));
3636
}
3737

3838
#ifndef PRODUCT
3939
void G1CardTable::verify_g1_young_region(MemRegion mr) {
40-
verify_region(mr, g1_young_gen, true);
40+
verify_region(mr, !UseNewCode ? g1_young_card_val() : dirty_card_val(), true);
4141
}
4242
#endif
4343

@@ -69,6 +69,7 @@ void G1CardTable::initialize(G1RegionToSpaceMapper* mapper) {
6969
}
7070

7171
bool G1CardTable::is_in_young(const void* p) const {
72-
volatile CardValue* card = byte_for(p);
73-
return *card == G1CardTable::g1_young_card_val();
72+
// volatile CardValue* card = byte_for(p);
73+
//return *card == G1CardTable::g1_young_card_val();
74+
return G1CollectedHeap::heap()->heap_region_containing(p)->is_young();
7475
}

0 commit comments

Comments
 (0)