Skip to content

Commit 3cfe7a4

Browse files
lxy-9602lxy-9602
authored
refactor: refactor sst and add io exception test (#188)
1 parent d51f047 commit 3cfe7a4

26 files changed

+235
-120
lines changed

‎cmake_modules/ThirdpartyToolchain.cmake‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,11 @@ set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
323323
set(EP_C_FLAGS "${CMAKE_C_FLAGS}")
324324
string(REPLACE "-Wglobal-constructors" "" EP_CXX_FLAGS ${EP_CXX_FLAGS})
325325
string(REPLACE "-Wglobal-constructors" "" EP_C_FLAGS ${EP_C_FLAGS})
326+
# Remove coverage flags from third-party dependencies to avoid gcov dependency
327+
string(REPLACE "--coverage" "" EP_CXX_FLAGS ${EP_CXX_FLAGS})
328+
string(REPLACE "--coverage" "" EP_C_FLAGS ${EP_C_FLAGS})
329+
string(REPLACE "-DCOVERAGE_BUILD" "" EP_CXX_FLAGS ${EP_CXX_FLAGS})
330+
string(REPLACE "-DCOVERAGE_BUILD" "" EP_C_FLAGS ${EP_C_FLAGS})
326331
if(NOT MSVC_TOOLCHAIN)
327332
# Set -fPIC on all external projects
328333
string(APPEND EP_CXX_FLAGS

‎src/paimon/common/data/serializer/row_compacted_serializer.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,8 @@ Result<MemorySlice::SliceComparator> RowCompactedSerializer::CreateSliceComparat
5959
auto comparator = [row_reader1, row_reader2, readers, comparators](
6060
const std::shared_ptr<MemorySlice>& slice1,
6161
const std::shared_ptr<MemorySlice>& slice2) -> Result<int32_t> {
62-
row_reader1->PointTo(*slice1->GetSegment(), slice1->Offset());
63-
row_reader2->PointTo(*slice2->GetSegment(), slice2->Offset());
62+
row_reader1->PointTo(slice1->GetSegment(), slice1->Offset());
63+
row_reader2->PointTo(slice2->GetSegment(), slice2->Offset());
6464
for (int32_t i = 0; i < static_cast<int32_t>(readers.size()); i++) {
6565
bool is_null1 = row_reader1->IsNullAt(i);
6666
bool is_null2 = row_reader2->IsNullAt(i);

‎src/paimon/common/io/cache/cache.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717
#include "paimon/common/io/cache/cache.h"
1818

1919
namespace paimon {
20-
std::shared_ptr<CacheValue> NoCache::Get(
20+
Result<std::shared_ptr<CacheValue>> NoCache::Get(
2121
const std::shared_ptr<CacheKey>& key,
22-
std::function<std::shared_ptr<CacheValue>(const std::shared_ptr<CacheKey>&)> supplier) {
22+
std::function<Result<std::shared_ptr<CacheValue>>(const std::shared_ptr<CacheKey>&)> supplier) {
2323
return supplier(key);
2424
}
2525

‎src/paimon/common/io/cache/cache.h‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,17 +22,18 @@
2222

2323
#include "paimon/common/io/cache/cache_key.h"
2424
#include "paimon/common/memory/memory_segment.h"
25-
#include "paimon/status.h"
25+
#include "paimon/result.h"
2626

2727
namespace paimon {
2828
class CacheValue;
2929

3030
class Cache {
3131
public:
3232
virtual ~Cache() = default;
33-
virtual std::shared_ptr<CacheValue> Get(
33+
virtual Result<std::shared_ptr<CacheValue>> Get(
3434
const std::shared_ptr<CacheKey>& key,
35-
std::function<std::shared_ptr<CacheValue>(const std::shared_ptr<CacheKey>&)> supplier) = 0;
35+
std::function<Result<std::shared_ptr<CacheValue>>(const std::shared_ptr<CacheKey>&)>
36+
supplier) = 0;
3637

3738
virtual void Put(const std::shared_ptr<CacheKey>& key,
3839
const std::shared_ptr<CacheValue>& value) = 0;
@@ -46,10 +47,10 @@ class Cache {
4647

4748
class NoCache : public Cache {
4849
public:
49-
std::shared_ptr<CacheValue> Get(
50+
Result<std::shared_ptr<CacheValue>> Get(
5051
const std::shared_ptr<CacheKey>& key,
51-
std::function<std::shared_ptr<CacheValue>(const std::shared_ptr<CacheKey>&)> supplier)
52-
override;
52+
std::function<Result<std::shared_ptr<CacheValue>>(const std::shared_ptr<CacheKey>&)>
53+
supplier) override;
5354
void Put(const std::shared_ptr<CacheKey>& key,
5455
const std::shared_ptr<CacheValue>& value) override;
5556
void Invalidate(const std::shared_ptr<CacheKey>& key) override;
@@ -59,13 +60,13 @@ class NoCache : public Cache {
5960

6061
class CacheValue {
6162
public:
62-
explicit CacheValue(const std::shared_ptr<MemorySegment>& segment) : segment_(segment) {}
63+
explicit CacheValue(const MemorySegment& segment) : segment_(segment) {}
6364

64-
std::shared_ptr<MemorySegment> GetSegment() {
65+
const MemorySegment& GetSegment() const {
6566
return segment_;
6667
}
6768

6869
private:
69-
std::shared_ptr<MemorySegment> segment_;
70+
MemorySegment segment_;
7071
};
7172
} // namespace paimon

‎src/paimon/common/io/cache/cache_manager.cpp‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,16 @@
1818

1919
namespace paimon {
2020

21-
std::shared_ptr<MemorySegment> CacheManager::GetPage(
21+
Result<MemorySegment> CacheManager::GetPage(
2222
std::shared_ptr<CacheKey>& key,
2323
std::function<Result<MemorySegment>(const std::shared_ptr<CacheKey>&)> reader) {
2424
auto& cache = key->IsIndex() ? index_cache_ : data_cache_;
25-
auto supplier = [=](const std::shared_ptr<CacheKey>& k) -> std::shared_ptr<CacheValue> {
26-
auto ret = reader(k);
27-
if (!ret.ok()) {
28-
return nullptr;
29-
}
30-
auto segment = ret.value();
31-
auto ptr = std::make_shared<MemorySegment>(segment);
32-
return std::make_shared<CacheValue>(ptr);
25+
auto supplier = [&](const std::shared_ptr<CacheKey>& k) -> Result<std::shared_ptr<CacheValue>> {
26+
PAIMON_ASSIGN_OR_RAISE(MemorySegment segment, reader(k));
27+
return std::make_shared<CacheValue>(segment);
3328
};
34-
return cache->Get(key, supplier)->GetSegment();
29+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<CacheValue> cache_value, cache->Get(key, supplier));
30+
return cache_value->GetSegment();
3531
}
3632

3733
void CacheManager::InvalidPage(const std::shared_ptr<CacheKey>& key) {

‎src/paimon/common/io/cache/cache_manager.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ class CacheManager {
3434
index_cache_ = std::make_shared<NoCache>();
3535
}
3636

37-
std::shared_ptr<MemorySegment> GetPage(
37+
Result<MemorySegment> GetPage(
3838
std::shared_ptr<CacheKey>& key,
3939
std::function<Result<MemorySegment>(const std::shared_ptr<CacheKey>&)> reader);
4040

‎src/paimon/common/io/memory_segment_output_stream.cpp‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,7 @@ MemorySegmentOutputStream::MemorySegmentOutputStream(int32_t segment_size,
3030
}
3131

3232
void MemorySegmentOutputStream::Advance() {
33-
MemorySegment next_segment = NextSegment();
34-
current_segment_ = std::make_shared<MemorySegment>(next_segment);
33+
current_segment_ = NextSegment();
3534
position_in_segment_ = 0;
3635
}
3736

@@ -61,7 +60,7 @@ void MemorySegmentOutputStream::Write(const char* data, uint32_t size) {
6160
void MemorySegmentOutputStream::Write(const MemorySegment& segment, int32_t offset, int32_t len) {
6261
int32_t remaining = segment_size_ - position_in_segment_;
6362
if (remaining >= len) {
64-
segment.CopyTo(offset, current_segment_.get(), position_in_segment_, len);
63+
segment.CopyTo(offset, &current_segment_, position_in_segment_, len);
6564
position_in_segment_ += len;
6665
} else {
6766
if (remaining == 0) {
@@ -70,7 +69,7 @@ void MemorySegmentOutputStream::Write(const MemorySegment& segment, int32_t offs
7069
}
7170
while (true) {
7271
int32_t to_put = std::min(remaining, len);
73-
segment.CopyTo(offset, current_segment_.get(), position_in_segment_, to_put);
72+
segment.CopyTo(offset, &current_segment_, position_in_segment_, to_put);
7473
offset += to_put;
7574
len -= to_put;
7675

‎src/paimon/common/io/memory_segment_output_stream.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ class PAIMON_EXPORT MemorySegmentOutputStream {
7979
int32_t segment_size_;
8080
int32_t position_in_segment_;
8181
std::shared_ptr<MemoryPool> pool_;
82-
std::shared_ptr<MemorySegment> current_segment_;
82+
MemorySegment current_segment_;
8383
std::vector<MemorySegment> memory_segments_;
8484

8585
ByteOrder byte_order_ = ByteOrder::PAIMON_BIG_ENDIAN;
@@ -88,7 +88,7 @@ class PAIMON_EXPORT MemorySegmentOutputStream {
8888
template <typename T>
8989
void MemorySegmentOutputStream::WriteValueImpl(T v) {
9090
if (position_in_segment_ <= segment_size_ - static_cast<int32_t>(sizeof(T))) {
91-
current_segment_->PutValue<T>(position_in_segment_, v);
91+
current_segment_.PutValue<T>(position_in_segment_, v);
9292
position_in_segment_ += sizeof(T);
9393
} else if (position_in_segment_ == segment_size_) {
9494
Advance();

‎src/paimon/common/lookup/lookup_store_factory.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@ Result<std::shared_ptr<BloomFilter>> LookupStoreFactory::BfGenerator(int64_t row
3434
return std::shared_ptr<BloomFilter>();
3535
}
3636
auto bloom_filter = BloomFilter::Create(row_count, options.GetLookupCacheBloomFilterFpp());
37-
auto bytes_for_bf = MemorySegment::AllocateHeapMemory(bloom_filter->ByteLength(), pool);
38-
auto memory_segment = std::make_shared<MemorySegment>(bytes_for_bf);
37+
MemorySegment memory_segment =
38+
MemorySegment::AllocateHeapMemory(bloom_filter->ByteLength(), pool);
3939
PAIMON_RETURN_NOT_OK(bloom_filter->SetMemorySegment(memory_segment));
4040
return bloom_filter;
4141
}

‎src/paimon/common/memory/memory_slice.cpp‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -21,16 +21,14 @@
2121
namespace paimon {
2222
std::shared_ptr<MemorySlice> MemorySlice::Wrap(const std::shared_ptr<Bytes>& bytes) {
2323
auto segment = MemorySegment::Wrap(bytes);
24-
auto ptr = std::make_shared<MemorySegment>(segment);
25-
return std::make_shared<MemorySlice>(ptr, 0, ptr->Size());
24+
return std::make_shared<MemorySlice>(segment, 0, segment.Size());
2625
}
2726

28-
std::shared_ptr<MemorySlice> MemorySlice::Wrap(const std::shared_ptr<MemorySegment>& segment) {
29-
return std::make_shared<MemorySlice>(segment, 0, segment->Size());
27+
std::shared_ptr<MemorySlice> MemorySlice::Wrap(const MemorySegment& segment) {
28+
return std::make_shared<MemorySlice>(segment, 0, segment.Size());
3029
}
3130

32-
MemorySlice::MemorySlice(const std::shared_ptr<MemorySegment>& segment, int32_t offset,
33-
int32_t length)
31+
MemorySlice::MemorySlice(const MemorySegment& segment, int32_t offset, int32_t length)
3432
: segment_(segment), offset_(offset), length_(length) {}
3533

3634
std::shared_ptr<MemorySlice> MemorySlice::Slice(int32_t index, int32_t length) {
@@ -49,38 +47,38 @@ int32_t MemorySlice::Offset() const {
4947
}
5048

5149
std::shared_ptr<Bytes> MemorySlice::GetHeapMemory() const {
52-
return segment_->GetHeapMemory();
50+
return segment_.GetHeapMemory();
5351
}
5452

55-
std::shared_ptr<MemorySegment> MemorySlice::GetSegment() const {
53+
const MemorySegment& MemorySlice::GetSegment() const {
5654
return segment_;
5755
}
5856

5957
int8_t MemorySlice::ReadByte(int32_t position) {
60-
return segment_->GetValue<int8_t>(offset_ + position);
58+
return segment_.GetValue<int8_t>(offset_ + position);
6159
}
6260

6361
int32_t MemorySlice::ReadInt(int32_t position) {
64-
return segment_->GetValue<int32_t>(offset_ + position);
62+
return segment_.GetValue<int32_t>(offset_ + position);
6563
}
6664

6765
int16_t MemorySlice::ReadShort(int32_t position) {
68-
return segment_->GetValue<int16_t>(offset_ + position);
66+
return segment_.GetValue<int16_t>(offset_ + position);
6967
}
7068

7169
int64_t MemorySlice::ReadLong(int32_t position) {
72-
return segment_->GetValue<int64_t>(offset_ + position);
70+
return segment_.GetValue<int64_t>(offset_ + position);
7371
}
7472

7573
std::string_view MemorySlice::ReadStringView() {
76-
auto array = segment_->GetArray();
74+
auto array = segment_.GetArray();
7775
return {array->data() + offset_, static_cast<size_t>(length_)};
7876
}
7977

8078
std::shared_ptr<Bytes> MemorySlice::CopyBytes(MemoryPool* pool) {
8179
auto bytes = std::make_shared<Bytes>(length_, pool);
8280
auto target = MemorySegment::Wrap(bytes);
83-
segment_->CopyTo(offset_, &target, 0, length_);
81+
segment_.CopyTo(offset_, &target, 0, length_);
8482
return bytes;
8583
}
8684

@@ -110,8 +108,8 @@ std::shared_ptr<MemorySliceInput> MemorySlice::ToInput() {
110108
int32_t MemorySlice::Compare(const MemorySlice& other) const {
111109
int32_t len = std::min(length_, other.length_);
112110
for (int32_t i = 0; i < len; ++i) {
113-
auto byte1 = static_cast<unsigned char>(segment_->Get(offset_ + i));
114-
auto byte2 = static_cast<unsigned char>(other.segment_->Get(other.offset_ + i));
111+
auto byte1 = static_cast<unsigned char>(segment_.Get(offset_ + i));
112+
auto byte2 = static_cast<unsigned char>(other.segment_.Get(other.offset_ + i));
115113
if (byte1 != byte2) {
116114
return static_cast<int>(byte1) - static_cast<int>(byte2);
117115
}

0 commit comments

Comments
 (0)