Skip to content

Commit 447b04f

Browse files
ChaomingZhangCNChaomingZhangCN
committed
check byte order
1 parent 9ab4a91 commit 447b04f

File tree

8 files changed

+80
-45
lines changed

8 files changed

+80
-45
lines changed

‎src/paimon/common/io/cache/cache.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ class CacheValue {
6565
return segment_;
6666
}
6767

68-
public:
68+
private:
6969
std::shared_ptr<MemorySegment> segment_;
7070
};
7171
} // namespace paimon

‎src/paimon/common/memory/memory_slice_input.cpp‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717
#include "paimon/common/memory/memory_slice_input.h"
1818

19+
#include "paimon/common/utils/math.h"
20+
1921
namespace paimon {
2022

2123
MemorySliceInput::MemorySliceInput(std::shared_ptr<MemorySlice>& slice)
@@ -52,6 +54,18 @@ int8_t MemorySliceInput::ReadUnsignedByte() {
5254
int32_t MemorySliceInput::ReadInt() {
5355
int v = slice_->ReadInt(position_);
5456
position_ += 4;
57+
if (NeedSwap()) {
58+
return EndianSwapValue(v);
59+
}
60+
return v;
61+
}
62+
63+
int64_t MemorySliceInput::ReadLong() {
64+
int64_t v = slice_->ReadLong(position_);
65+
position_ += 8;
66+
if (NeedSwap()) {
67+
return EndianSwapValue(v);
68+
}
5569
return v;
5670
}
5771

@@ -66,12 +80,6 @@ int32_t MemorySliceInput::ReadVarLenInt() {
6680
throw std::invalid_argument("Malformed integer.");
6781
}
6882

69-
int64_t MemorySliceInput::ReadLong() {
70-
int64_t v = slice_->ReadLong(position_);
71-
position_ += 8;
72-
return v;
73-
}
74-
7583
int64_t MemorySliceInput::ReadVarLenLong() {
7684
int64_t result = 0;
7785
for (int offset = 0; offset < 64; offset += 7) {
@@ -84,6 +92,14 @@ int64_t MemorySliceInput::ReadVarLenLong() {
8492
throw std::invalid_argument("Malformed long.");
8593
}
8694

95+
void MemorySliceInput::SetOrder(ByteOrder order) {
96+
byte_order_ = order;
97+
}
98+
99+
bool MemorySliceInput::NeedSwap() const {
100+
return SystemByteOrder() != byte_order_;
101+
}
102+
87103
std::shared_ptr<MemorySlice> MemorySliceInput::ReadSlice(int length) {
88104
auto slice = slice_->Slice(position_, length);
89105
position_ += length;

‎src/paimon/common/memory/memory_slice_input.h‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include <type_traits>
2525

2626
#include "paimon/common/memory/memory_slice.h"
27+
#include "paimon/io/byte_order.h"
2728
#include "paimon/status.h"
2829
#include "paimon/visibility.h"
2930

@@ -46,14 +47,21 @@ class PAIMON_EXPORT MemorySliceInput {
4647
int8_t ReadByte();
4748
int8_t ReadUnsignedByte();
4849
int32_t ReadInt();
49-
int32_t ReadVarLenInt();
5050
int64_t ReadLong();
51+
int32_t ReadVarLenInt();
5152
int64_t ReadVarLenLong();
5253
std::shared_ptr<MemorySlice> ReadSlice(int length);
5354

55+
void SetOrder(ByteOrder order);
56+
57+
private:
58+
bool NeedSwap() const;
59+
5460
private:
5561
std::shared_ptr<MemorySlice> slice_;
5662
int32_t position_;
63+
64+
ByteOrder byte_order_ = ByteOrder::PAIMON_BIG_ENDIAN;
5765
};
5866

5967
} // namespace paimon

‎src/paimon/common/memory/memory_slice_output.cpp‎

Lines changed: 30 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
#include "paimon/common/memory/memory_slice_output.h"
1818

19+
#include "paimon/common/utils/math.h"
1920
namespace paimon {
2021

2122
MemorySliceOutput::MemorySliceOutput(int32_t estimated_size, MemoryPool* pool) {
@@ -37,49 +38,38 @@ std::unique_ptr<MemorySlice> MemorySliceOutput::ToSlice() {
3738
return std::make_unique<MemorySlice>(segment, 0, size_);
3839
}
3940

40-
void MemorySliceOutput::WriteByte(int8_t value) {
41-
EnsureSize(size_ + 1);
42-
segment_.Put(size_++, static_cast<char>(value));
43-
}
44-
45-
void MemorySliceOutput::WriteShort(int16_t value) {
46-
EnsureSize(size_ + 2);
47-
segment_.PutValue(size_, value);
48-
size_ += 2;
49-
}
50-
51-
void MemorySliceOutput::WriteInt(int32_t value) {
52-
EnsureSize(size_ + 4);
53-
segment_.PutValue(size_, value);
54-
size_ += 4;
55-
}
56-
57-
void MemorySliceOutput::WriteLong(int64_t value) {
58-
EnsureSize(size_ + 8);
59-
segment_.PutValue(size_, value);
60-
size_ += 8;
41+
template <typename T>
42+
void MemorySliceOutput::WriteValue(T value) {
43+
int32_t write_length = sizeof(T);
44+
EnsureSize(size_ + write_length);
45+
T write_value = value;
46+
if (NeedSwap()) {
47+
write_value = EndianSwapValue(value);
48+
}
49+
segment_.PutValue(size_, write_value);
50+
size_ += write_length;
6151
}
6252

6353
void MemorySliceOutput::WriteVarLenInt(int32_t value) {
6454
if (value < 0) {
6555
throw std::invalid_argument("negative value: v=" + std::to_string(value));
6656
}
6757
while ((value & ~0x7F) != 0) {
68-
WriteByte(((value & 0x7F) | 0x80));
58+
WriteValue(static_cast<char>((value & 0x7F) | 0x80));
6959
value >>= 7;
7060
}
71-
WriteByte(static_cast<int8_t>(value));
61+
WriteValue(static_cast<char>(value));
7262
}
7363

7464
void MemorySliceOutput::WriteVarLenLong(int64_t value) {
7565
if (value < 0) {
7666
throw std::invalid_argument("negative value: v=" + std::to_string(value));
7767
}
7868
while ((value & ~0x7F) != 0) {
79-
WriteByte(((value & 0x7F) | 0x80));
69+
WriteValue(static_cast<char>((value & 0x7F) | 0x80));
8070
value >>= 7;
8171
}
82-
WriteByte(static_cast<int8_t>(value));
72+
WriteValue(static_cast<char>(value));
8373
}
8474

8575
void MemorySliceOutput::WriteBytes(const std::shared_ptr<Bytes>& source) {
@@ -94,6 +84,14 @@ void MemorySliceOutput::WriteBytes(const std::shared_ptr<Bytes>& source, int sou
9484
size_ += length;
9585
}
9686

87+
void MemorySliceOutput::SetOrder(ByteOrder order) {
88+
byte_order_ = order;
89+
}
90+
91+
bool MemorySliceOutput::NeedSwap() const {
92+
return SystemByteOrder() != byte_order_;
93+
}
94+
9795
void MemorySliceOutput::EnsureSize(int size) {
9896
if (size <= segment_.Size()) {
9997
return;
@@ -111,4 +109,11 @@ void MemorySliceOutput::EnsureSize(int size) {
111109
segment_ = new_segment;
112110
}
113111

112+
template void MemorySliceOutput::WriteValue(bool);
113+
template void MemorySliceOutput::WriteValue(char);
114+
template void MemorySliceOutput::WriteValue(int8_t);
115+
template void MemorySliceOutput::WriteValue(int16_t);
116+
template void MemorySliceOutput::WriteValue(int32_t);
117+
template void MemorySliceOutput::WriteValue(int64_t);
118+
114119
} // namespace paimon

‎src/paimon/common/memory/memory_slice_output.h‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include <type_traits>
2525

2626
#include "paimon/common/memory/memory_slice.h"
27+
#include "paimon/io/byte_order.h"
2728
#include "paimon/visibility.h"
2829

2930
namespace paimon {
@@ -40,22 +41,27 @@ class PAIMON_EXPORT MemorySliceOutput {
4041
void Reset();
4142
std::unique_ptr<MemorySlice> ToSlice();
4243

43-
void WriteByte(int8_t value);
44-
void WriteShort(int16_t value);
45-
void WriteInt(int32_t value);
44+
template <typename T>
45+
void WriteValue(T value);
46+
4647
void WriteVarLenInt(int32_t value);
47-
void WriteLong(int64_t value);
4848
void WriteVarLenLong(int64_t value);
49+
4950
void WriteBytes(const std::shared_ptr<Bytes>& source);
5051
void WriteBytes(const std::shared_ptr<Bytes>& source, int source_index, int length);
5152

53+
void SetOrder(ByteOrder order);
54+
5255
private:
5356
void EnsureSize(int bytes);
57+
bool NeedSwap() const;
5458

5559
private:
5660
MemoryPool* pool_;
5761
MemorySegment segment_;
5862
int32_t size_;
63+
64+
ByteOrder byte_order_ = ByteOrder::PAIMON_BIG_ENDIAN;
5965
};
6066

6167
} // namespace paimon

‎src/paimon/common/sst/block_cache.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ class BlockCache {
6161
return segment;
6262
}
6363

64-
public:
64+
private:
6565
std::string file_path_;
6666
std::shared_ptr<InputStream> in_;
6767
std::shared_ptr<MemoryPool> pool_;

‎src/paimon/common/sst/block_trailer.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ std::string BlockTrailer::ToString() const {
4242

4343
std::shared_ptr<MemorySlice> BlockTrailer::WriteBlockTrailer(MemoryPool* pool) {
4444
auto output = std::make_shared<MemorySliceOutput>(ENCODED_LENGTH, pool);
45-
output->WriteByte(compression_type_);
46-
output->WriteInt(crc32c_);
45+
output->WriteValue(compression_type_);
46+
output->WriteValue(crc32c_);
4747
return output->ToSlice();
4848
}
4949
} // namespace paimon

‎src/paimon/common/sst/block_writer.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,15 +52,15 @@ Result<std::unique_ptr<paimon::MemorySlice>> BlockWriter::Finish() {
5252
aligned_ = false;
5353
}
5454
if (aligned_) {
55-
block_->WriteInt(aligned_size_);
55+
block_->WriteValue(aligned_size_);
5656
} else {
5757
for (auto& position : positions_) {
58-
block_->WriteInt(position);
58+
block_->WriteValue(position);
5959
}
60-
block_->WriteInt(positions_.size());
60+
block_->WriteValue(static_cast<int32_t>(positions_.size()));
6161
}
62-
block_->WriteByte(aligned_ ? static_cast<int8_t>(BlockAlignedType::ALIGNED)
63-
: static_cast<int8_t>(BlockAlignedType::UNALIGNED));
62+
block_->WriteValue(aligned_ ? static_cast<char>(BlockAlignedType::ALIGNED)
63+
: static_cast<char>(BlockAlignedType::UNALIGNED));
6464
return block_->ToSlice();
6565
}
6666

0 commit comments

Comments
 (0)