Skip to content

Commit 4ee35ee

Browse files
rmacnak-googleCommit Bot
rmacnak-google
authored and
Commit Bot
committed
[vm, compiler] Load unboxed doubles directly from the literal pool.
TEST=ci Change-Id: I4f881bc3e29059bce09eb0aa0ba99928e7fb10a8 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/244120 Reviewed-by: Alexander Aprelev <[email protected]> Commit-Queue: Ryan Macnak <[email protected]>
1 parent 34bca35 commit 4ee35ee

File tree

11 files changed

+183
-45
lines changed

11 files changed

+183
-45
lines changed

‎runtime/vm/compiler/assembler/assembler_arm.cc

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1541,10 +1541,6 @@ void Assembler::Drop(intptr_t stack_elements) {
15411541
}
15421542
}
15431543

1544-
intptr_t Assembler::FindImmediate(int32_t imm) {
1545-
return object_pool_builder().FindImmediate(imm);
1546-
}
1547-
15481544
// Uses a code sequence that can easily be decoded.
15491545
void Assembler::LoadWordFromPoolIndex(Register rd,
15501546
intptr_t index,
@@ -2840,7 +2836,15 @@ void Assembler::LoadDImmediate(DRegister dd,
28402836
Condition cond) {
28412837
ASSERT(scratch != PC);
28422838
ASSERT(scratch != IP);
2843-
if (!vmovd(dd, value, cond)) {
2839+
if (vmovd(dd, value, cond)) return;
2840+
2841+
int64_t imm64 = bit_cast<int64_t, double>(value);
2842+
if (constant_pool_allowed()) {
2843+
intptr_t index = object_pool_builder().FindImmediate64(imm64);
2844+
intptr_t offset =
2845+
target::ObjectPool::element_offset(index) - kHeapObjectTag;
2846+
LoadDFromOffset(dd, PP, offset, cond);
2847+
} else {
28442848
// A scratch register and IP are needed to load an arbitrary double.
28452849
ASSERT(scratch != kNoRegister);
28462850
int64_t imm64 = bit_cast<int64_t, double>(value);
@@ -2850,6 +2854,13 @@ void Assembler::LoadDImmediate(DRegister dd,
28502854
}
28512855
}
28522856

2857+
void Assembler::LoadQImmediate(QRegister qd, simd128_value_t value) {
2858+
ASSERT(constant_pool_allowed());
2859+
intptr_t index = object_pool_builder().FindImmediate128(value);
2860+
intptr_t offset = target::ObjectPool::element_offset(index) - kHeapObjectTag;
2861+
LoadMultipleDFromOffset(EvenDRegisterOf(qd), 2, PP, offset);
2862+
}
2863+
28532864
void Assembler::LoadFromOffset(Register reg,
28542865
const Address& address,
28552866
OperandSize size,

‎runtime/vm/compiler/assembler/assembler_arm.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -874,6 +874,7 @@ class Assembler : public AssemblerBase {
874874
double value,
875875
Register scratch,
876876
Condition cond = AL);
877+
void LoadQImmediate(QRegister dd, simd128_value_t value);
877878

878879
void MarkExceptionHandler(Label* label);
879880

@@ -985,7 +986,6 @@ class Assembler : public AssemblerBase {
985986
Register scratch,
986987
bool can_be_null = false) override;
987988

988-
intptr_t FindImmediate(int32_t imm);
989989
bool CanLoadFromObjectPool(const Object& object) const;
990990
void LoadFromOffset(Register reg,
991991
const Address& address,

‎runtime/vm/compiler/assembler/assembler_arm64.cc

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -476,10 +476,6 @@ void Assembler::LoadDoubleWordFromPoolIndex(Register lower,
476476
}
477477
}
478478

479-
intptr_t Assembler::FindImmediate(int64_t imm) {
480-
return object_pool_builder().FindImmediate(imm);
481-
}
482-
483479
bool Assembler::CanLoadFromObjectPool(const Object& object) const {
484480
ASSERT(IsOriginalObject(object));
485481
if (!constant_pool_allowed()) {
@@ -643,7 +639,7 @@ void Assembler::LoadImmediate(Register reg, int64_t imm) {
643639

644640
// Use constant pool if allowed, unless we can load imm with 2 instructions.
645641
if ((w1 != 0) && constant_pool_allowed()) {
646-
const intptr_t index = FindImmediate(imm);
642+
const intptr_t index = object_pool_builder().FindImmediate(imm);
647643
LoadWordFromPoolIndex(reg, index);
648644
return;
649645
}
@@ -679,13 +675,26 @@ void Assembler::LoadImmediate(Register reg, int64_t imm) {
679675
}
680676

681677
void Assembler::LoadDImmediate(VRegister vd, double immd) {
682-
if (!fmovdi(vd, immd)) {
683-
int64_t imm = bit_cast<int64_t, double>(immd);
684-
LoadImmediate(TMP, imm);
678+
if (fmovdi(vd, immd)) return;
679+
680+
int64_t imm64 = bit_cast<int64_t, double>(immd);
681+
if (constant_pool_allowed()) {
682+
intptr_t index = object_pool_builder().FindImmediate64(imm64);
683+
intptr_t offset = target::ObjectPool::element_offset(index);
684+
LoadDFromOffset(vd, PP, offset);
685+
} else {
686+
LoadImmediate(TMP, imm64);
685687
fmovdr(vd, TMP);
686688
}
687689
}
688690

691+
void Assembler::LoadQImmediate(VRegister vd, simd128_value_t immq) {
692+
ASSERT(constant_pool_allowed());
693+
intptr_t index = object_pool_builder().FindImmediate128(immq);
694+
intptr_t offset = target::ObjectPool::element_offset(index);
695+
LoadQFromOffset(vd, PP, offset);
696+
}
697+
689698
void Assembler::Branch(const Code& target,
690699
Register pp,
691700
ObjectPoolBuilderEntry::Patchability patchable) {

‎runtime/vm/compiler/assembler/assembler_arm64.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2041,7 +2041,6 @@ class Assembler : public AssemblerBase {
20412041
compiler::LRState lr_state() const { return lr_state_; }
20422042
void set_lr_state(compiler::LRState state) { lr_state_ = state; }
20432043

2044-
intptr_t FindImmediate(int64_t imm);
20452044
bool CanLoadFromObjectPool(const Object& object) const;
20462045
void LoadNativeEntry(Register dst,
20472046
const ExternalLabel* label,
@@ -2057,6 +2056,7 @@ class Assembler : public AssemblerBase {
20572056
void LoadImmediate(Register reg, int64_t imm);
20582057

20592058
void LoadDImmediate(VRegister reg, double immd);
2059+
void LoadQImmediate(VRegister reg, simd128_value_t immq);
20602060

20612061
// Load word from pool from the given offset using encoding that
20622062
// InstructionPattern::DecodeLoadWordFromPool can decode.

‎runtime/vm/compiler/assembler/assembler_base.cc

Lines changed: 81 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -309,11 +309,25 @@ void AssemblerBase::Stop(const char* message) {
309309
}
310310

311311
uword ObjIndexPair::Hash(Key key) {
312-
if (key.type() != ObjectPoolBuilderEntry::kTaggedObject) {
313-
return key.raw_value_;
312+
switch (key.type()) {
313+
case ObjectPoolBuilderEntry::kImmediate128:
314+
return key.imm128_.int_storage[0] ^ key.imm128_.int_storage[1] ^
315+
key.imm128_.int_storage[2] ^ key.imm128_.int_storage[3];
316+
317+
#if defined(TARGET_ARCH_IS_32_BIT)
318+
case ObjectPoolBuilderEntry::kImmediate64:
319+
return key.imm64_;
320+
#endif
321+
case ObjectPoolBuilderEntry::kImmediate:
322+
case ObjectPoolBuilderEntry::kNativeFunction:
323+
case ObjectPoolBuilderEntry::kSwitchableCallMissEntryPoint:
324+
case ObjectPoolBuilderEntry::kMegamorphicCallEntryPoint:
325+
return key.imm_;
326+
case ObjectPoolBuilderEntry::kTaggedObject:
327+
return ObjectHash(*key.obj_);
314328
}
315329

316-
return ObjectHash(*key.obj_);
330+
UNREACHABLE();
317331
}
318332

319333
void ObjectPoolBuilder::Reset() {
@@ -342,6 +356,22 @@ intptr_t ObjectPoolBuilder::AddImmediate(uword imm) {
342356
ObjectPoolBuilderEntry::kNotPatchable));
343357
}
344358

359+
intptr_t ObjectPoolBuilder::AddImmediate64(uint64_t imm) {
360+
#if defined(TARGET_ARCH_IS_32_BIT)
361+
return AddObject(
362+
ObjectPoolBuilderEntry(imm, ObjectPoolBuilderEntry::kImmediate64,
363+
ObjectPoolBuilderEntry::kNotPatchable));
364+
#else
365+
return AddImmediate(imm);
366+
#endif
367+
}
368+
369+
intptr_t ObjectPoolBuilder::AddImmediate128(simd128_value_t imm) {
370+
return AddObject(
371+
ObjectPoolBuilderEntry(imm, ObjectPoolBuilderEntry::kImmediate128,
372+
ObjectPoolBuilderEntry::kNotPatchable));
373+
}
374+
345375
intptr_t ObjectPoolBuilder::AddObject(ObjectPoolBuilderEntry entry) {
346376
ASSERT((entry.type() != ObjectPoolBuilderEntry::kTaggedObject) ||
347377
(IsNotTemporaryScopedHandle(*entry.obj_) &&
@@ -359,6 +389,38 @@ intptr_t ObjectPoolBuilder::AddObject(ObjectPoolBuilderEntry entry) {
359389
}
360390
}
361391

392+
#if defined(TARGET_ARCH_IS_32_BIT)
393+
if (entry.type() == ObjectPoolBuilderEntry::kImmediate64) {
394+
ASSERT(entry.patchable() == ObjectPoolBuilderEntry::kNotPatchable);
395+
uint64_t imm = entry.imm64_;
396+
intptr_t idx = AddImmediate(Utils::Low32Bits(imm));
397+
AddImmediate(Utils::High32Bits(imm));
398+
object_pool_index_table_.Insert(ObjIndexPair(entry, idx));
399+
return idx;
400+
}
401+
if (entry.type() == ObjectPoolBuilderEntry::kImmediate128) {
402+
ASSERT(entry.patchable() == ObjectPoolBuilderEntry::kNotPatchable);
403+
intptr_t idx = AddImmediate(entry.imm128_.int_storage[0]);
404+
AddImmediate(entry.imm128_.int_storage[1]);
405+
AddImmediate(entry.imm128_.int_storage[2]);
406+
AddImmediate(entry.imm128_.int_storage[3]);
407+
object_pool_index_table_.Insert(ObjIndexPair(entry, idx));
408+
return idx;
409+
}
410+
#else
411+
if (entry.type() == ObjectPoolBuilderEntry::kImmediate128) {
412+
ASSERT(entry.patchable() == ObjectPoolBuilderEntry::kNotPatchable);
413+
uword lo64 = static_cast<uword>(entry.imm128_.int_storage[0]) |
414+
(static_cast<uword>(entry.imm128_.int_storage[1]) << 32);
415+
uword hi64 = static_cast<uword>(entry.imm128_.int_storage[2]) |
416+
(static_cast<uword>(entry.imm128_.int_storage[3]) << 32);
417+
intptr_t idx = AddImmediate(lo64);
418+
AddImmediate(hi64);
419+
object_pool_index_table_.Insert(ObjIndexPair(entry, idx));
420+
return idx;
421+
}
422+
#endif
423+
362424
const intptr_t idx = base_index_ + object_pool_.length();
363425
object_pool_.Add(entry);
364426
if (entry.patchable() == ObjectPoolBuilderEntry::kNotPatchable) {
@@ -407,6 +469,22 @@ intptr_t ObjectPoolBuilder::FindImmediate(uword imm) {
407469
ObjectPoolBuilderEntry::kNotPatchable));
408470
}
409471

472+
intptr_t ObjectPoolBuilder::FindImmediate64(uint64_t imm) {
473+
#if defined(TARGET_ARCH_IS_32_BIT)
474+
return FindObject(
475+
ObjectPoolBuilderEntry(imm, ObjectPoolBuilderEntry::kImmediate64,
476+
ObjectPoolBuilderEntry::kNotPatchable));
477+
#else
478+
return FindImmediate(imm);
479+
#endif
480+
}
481+
482+
intptr_t ObjectPoolBuilder::FindImmediate128(simd128_value_t imm) {
483+
return FindObject(
484+
ObjectPoolBuilderEntry(imm, ObjectPoolBuilderEntry::kImmediate128,
485+
ObjectPoolBuilderEntry::kNotPatchable));
486+
}
487+
410488
intptr_t ObjectPoolBuilder::FindNativeFunction(
411489
const ExternalLabel* label,
412490
ObjectPoolBuilderEntry::Patchability patchable) {

‎runtime/vm/compiler/assembler/assembler_riscv.cc

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3269,9 +3269,6 @@ void Assembler::LoadPoolPointer(Register pp) {
32693269
set_constant_pool_allowed(pp == PP);
32703270
}
32713271

3272-
intptr_t Assembler::FindImmediate(int64_t imm) {
3273-
UNIMPLEMENTED();
3274-
}
32753272
bool Assembler::CanLoadFromObjectPool(const Object& object) const {
32763273
ASSERT(IsOriginalObject(object));
32773274
if (!constant_pool_allowed()) {
@@ -3336,21 +3333,16 @@ void Assembler::LoadDImmediate(FRegister reg, double immd) {
33363333
#endif
33373334
} else {
33383335
ASSERT(constant_pool_allowed());
3339-
#if XLEN >= 64
3340-
intptr_t index = object_pool_builder().FindImmediate(imm);
3336+
intptr_t index = object_pool_builder().FindImmediate64(imm);
33413337
intptr_t offset = target::ObjectPool::element_offset(index);
3342-
#else
3343-
intptr_t lo_index =
3344-
object_pool_builder().AddImmediate(Utils::Low32Bits(imm));
3345-
intptr_t hi_index =
3346-
object_pool_builder().AddImmediate(Utils::High32Bits(imm));
3347-
ASSERT(lo_index + 1 == hi_index);
3348-
intptr_t offset = target::ObjectPool::element_offset(lo_index);
3349-
#endif
33503338
LoadDFromOffset(reg, PP, offset);
33513339
}
33523340
}
33533341

3342+
void Assembler::LoadQImmediate(FRegister reg, simd128_value_t immq) {
3343+
UNREACHABLE(); // F registers cannot represent SIMD128.
3344+
}
3345+
33543346
// Load word from pool from the given offset using encoding that
33553347
// InstructionPattern::DecodeLoadWordFromPool can decode.
33563348
//

‎runtime/vm/compiler/assembler/assembler_riscv.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1160,7 +1160,6 @@ class Assembler : public MicroAssembler {
11601160
bool constant_pool_allowed() const { return constant_pool_allowed_; }
11611161
void set_constant_pool_allowed(bool b) { constant_pool_allowed_ = b; }
11621162

1163-
intptr_t FindImmediate(int64_t imm);
11641163
bool CanLoadFromObjectPool(const Object& object) const;
11651164
void LoadNativeEntry(Register dst,
11661165
const ExternalLabel* label,
@@ -1180,6 +1179,7 @@ class Assembler : public MicroAssembler {
11801179
void LoadImmediate(Register reg, intx_t imm);
11811180

11821181
void LoadDImmediate(FRegister reg, double immd);
1182+
void LoadQImmediate(FRegister reg, simd128_value_t immq);
11831183

11841184
// Load word from pool from the given offset using encoding that
11851185
// InstructionPattern::DecodeLoadWordFromPool can decode.

‎runtime/vm/compiler/assembler/assembler_x64.cc

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1381,17 +1381,13 @@ void Assembler::CompareObject(Register reg, const Object& object) {
13811381
}
13821382
}
13831383

1384-
intptr_t Assembler::FindImmediate(int64_t imm) {
1385-
return object_pool_builder().FindImmediate(imm);
1386-
}
1387-
13881384
void Assembler::LoadImmediate(Register reg, const Immediate& imm) {
13891385
if (imm.value() == 0) {
13901386
xorl(reg, reg);
13911387
} else if (imm.is_int32() || !constant_pool_allowed()) {
13921388
movq(reg, imm);
13931389
} else {
1394-
const intptr_t idx = FindImmediate(imm.value());
1390+
const intptr_t idx = object_pool_builder().FindImmediate(imm.value());
13951391
LoadWordFromPoolIndex(reg, idx);
13961392
}
13971393
}
@@ -1410,12 +1406,18 @@ void Assembler::LoadDImmediate(FpuRegister dst, double immediate) {
14101406
if (bits == 0) {
14111407
xorps(dst, dst);
14121408
} else {
1413-
intptr_t index = FindImmediate(bits);
1409+
intptr_t index = object_pool_builder().FindImmediate64(bits);
14141410
LoadUnboxedDouble(
14151411
dst, PP, target::ObjectPool::element_offset(index) - kHeapObjectTag);
14161412
}
14171413
}
14181414

1415+
void Assembler::LoadQImmediate(FpuRegister dst, simd128_value_t immediate) {
1416+
intptr_t index = object_pool_builder().FindImmediate128(immediate);
1417+
movups(dst, Address(PP, target::ObjectPool::element_offset(index) -
1418+
kHeapObjectTag));
1419+
}
1420+
14191421
void Assembler::LoadCompressed(Register dest, const Address& slot) {
14201422
#if !defined(DART_COMPRESSED_POINTERS)
14211423
movq(dest, slot);

‎runtime/vm/compiler/assembler/assembler_x64.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -781,6 +781,7 @@ class Assembler : public AssemblerBase {
781781
LoadImmediate(reg, Immediate(immediate));
782782
}
783783
void LoadDImmediate(FpuRegister dst, double immediate);
784+
void LoadQImmediate(FpuRegister dst, simd128_value_t immediate);
784785

785786
void LoadIsolate(Register dst);
786787
void LoadIsolateGroup(Register dst);
@@ -1316,7 +1317,6 @@ class Assembler : public AssemblerBase {
13161317
private:
13171318
bool constant_pool_allowed_;
13181319

1319-
intptr_t FindImmediate(int64_t imm);
13201320
bool CanLoadFromObjectPool(const Object& object) const;
13211321
void LoadObjectHelper(Register dst, const Object& obj, bool is_unique);
13221322
void LoadWordFromPoolIndex(Register dst, intptr_t index);

0 commit comments

Comments
 (0)