Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove serialization non C.41 constructors from fragment_metadata_from_capnp #4682

Draft
wants to merge 22 commits into
base: main
Choose a base branch
from
Draft

Remove serialization non C.41 constructors from fragment_metadata_from_capnp #4682

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
8c00ca3
Unstatus `deserialize_non_empty_domain` and make it non-generic.
teo-tsirpanis Jan 29, 2024
a6f0b29
Unstatus `serialize_non_empty_domain` and make it non-generic.
teo-tsirpanis Jan 29, 2024
2767d48
Add C.41 constructor for FragmentMetadata and use it in serialization.
teo-tsirpanis Jan 29, 2024
a2b1933
Fix compile errors.
teo-tsirpanis Jan 29, 2024
f2728ba
Clean-up `FragmentMetadata`: remove unused methods, add missing const…
teo-tsirpanis Jan 29, 2024
9a3ab85
Fix compile errors.
teo-tsirpanis Jan 30, 2024
9f87c6a
Fix tests.
teo-tsirpanis Jan 30, 2024
a609b10
Fix more tests.
teo-tsirpanis Jan 30, 2024
2d8b4ea
Move `UnorderedWriter::resources` to `StrategyBase`.
teo-tsirpanis Jan 30, 2024
3aec56d
Fix segfaults by passing context resources when deserializing fragmen…
teo-tsirpanis Jan 30, 2024
c9ffdb0
clang-format
teo-tsirpanis Jan 30, 2024
43d8738
Address PR feedback.
teo-tsirpanis Feb 13, 2024
d6af715
Merge branch 'dev' into teo/fragment-metadata-c.41
teo-tsirpanis Feb 13, 2024
0ad881e
Fix most compile errors.
teo-tsirpanis Feb 15, 2024
2f29592
Address some PR feedback.
teo-tsirpanis Feb 15, 2024
c5d8c38
Merge branch 'dev' into teo/fragment-metadata-c.41
teo-tsirpanis Apr 17, 2024
f894873
Restore some deleted move constructors.
teo-tsirpanis Apr 17, 2024
bcedcbe
Fix some errors.
teo-tsirpanis Apr 17, 2024
d4d9007
Fix compile errors.
teo-tsirpanis Apr 29, 2024
51902e9
Merge branch 'dev' into teo/fragment-metadata-c.41
teo-tsirpanis Apr 29, 2024
491ebf5
Instrument fragment mins/maxs.
teo-tsirpanis Apr 29, 2024
790e370
Merge branch 'dev' into teo/fragment-metadata-c.41
teo-tsirpanis May 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions test/src/unit-tile-metadata.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1554,7 +1554,9 @@ struct CPPTileMetadataStringDimFx {
// Validate mins.
auto& min = frag_meta[0]->get_min("a");
CHECK(min.size() == sizeof(double));
CHECK(*static_cast<double*>(static_cast<void*>(min.data())) == 4);
CHECK(
*static_cast<const double*>(static_cast<const void*>(min.data())) ==
4);

CHECK_THROWS_WITH(
frag_meta[0]->get_min("d1"),
Expand All @@ -1569,7 +1571,9 @@ struct CPPTileMetadataStringDimFx {
// Validate maxs.
auto& max = frag_meta[0]->get_max("a");
CHECK(max.size() == sizeof(double));
CHECK(*static_cast<double*>(static_cast<void*>(max.data())) == 7);
CHECK(
*static_cast<const double*>(static_cast<const void*>(max.data())) ==
7);

CHECK_THROWS_WITH(
frag_meta[0]->get_max("d1"),
Expand Down
124 changes: 87 additions & 37 deletions tiledb/sm/fragment/fragment_metadata.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,87 @@ FragmentMetadata::FragmentMetadata(
array_schema_name_ = array_schema_->name();
}

FragmentMetadata::FragmentMetadata(
ContextResources* resources,
MemoryTracker* memory_tracker,
shared_ptr<const ArraySchema> array_schema,
std::vector<uint64_t>&& file_sizes,
std::vector<uint64_t>&& file_var_sizes,
std::vector<uint64_t>&& file_validity_sizes,
URI&& fragment_uri,
bool has_timestamps,
bool has_delete_meta,
bool has_consolidated_footer,
uint64_t sparse_tile_num,
uint64_t tile_index_base,
std::vector<std::vector<uint64_t>>&& tile_offsets,
std::vector<std::vector<uint64_t>>&& tile_var_offsets,
std::vector<std::vector<uint64_t>>&& tile_var_sizes,
std::vector<std::vector<uint64_t>>&& tile_validity_offsets,
std::vector<std::vector<uint8_t>>&& tile_min_buffer,
std::vector<std::vector<char>>&& tile_min_var_buffer,
std::vector<std::vector<uint8_t>>&& tile_max_buffer,
std::vector<std::vector<char>>&& tile_max_var_buffer,
std::vector<std::vector<uint8_t>>&& tile_sums,
std::vector<std::vector<uint64_t>>&& tile_null_counts,
std::vector<std::vector<uint8_t>>&& fragment_mins,
std::vector<std::vector<uint8_t>>&& fragment_maxs,
std::vector<uint64_t>&& fragment_sums,
std::vector<uint64_t>&& fragment_null_counts,
uint32_t version,
std::pair<uint64_t, uint64_t> timestamp_range,
uint64_t last_tile_cell_num,
NDRange&& non_empty_domain,
RTree&& rtree,
FragmentMetadata::GenericTileOffsets&& generic_tile_offsets,
FragmentMetadata::LoadedMetadata&& loaded_metadata)
: resources_(resources)
, memory_tracker_(memory_tracker)
, array_schema_(array_schema)
, array_schema_name_(array_schema->name())
, dense_(array_schema_->dense())
, file_sizes_(std::move(file_sizes))
, file_var_sizes_(std::move(file_var_sizes))
, file_validity_sizes_(std::move(file_validity_sizes))
, footer_size_(0)
, footer_offset_(0)
, fragment_uri_(std::move(fragment_uri))
, has_consolidated_footer_(has_consolidated_footer)
, last_tile_cell_num_(last_tile_cell_num)
, has_timestamps_(has_timestamps)
, has_delete_meta_(has_delete_meta)
, sparse_tile_num_(sparse_tile_num)
, loaded_metadata_(std::move(loaded_metadata))
, meta_file_size_(0)
, tile_offsets_mtx_(tile_offsets.size())
, tile_var_offsets_mtx_(tile_var_offsets.size())
, rtree_(std::move(rtree))
, tile_index_base_(tile_index_base)
, tile_offsets_(std::move(tile_offsets))
, tile_var_offsets_(std::move(tile_var_offsets))
, tile_var_sizes_(std::move(tile_var_sizes))
, tile_validity_offsets_(std::move(tile_validity_offsets))
, tile_min_buffer_(std::move(tile_min_buffer))
, tile_min_var_buffer_(std::move(tile_min_var_buffer))
, tile_max_buffer_(std::move(tile_max_buffer))
, tile_max_var_buffer_(std::move(tile_max_var_buffer))
, tile_sums_(std::move(tile_sums))
, tile_null_counts_(std::move(tile_null_counts))
teo-tsirpanis marked this conversation as resolved.
Show resolved Hide resolved
, fragment_mins_(std::move(fragment_mins))
, fragment_maxs_(std::move(fragment_maxs))
, fragment_sums_(std::move(fragment_sums))
, fragment_null_counts_(std::move(fragment_null_counts))
, version_(version)
, timestamp_range_(timestamp_range)
, gt_offsets_(std::move(generic_tile_offsets)) {
build_idx_map();
if (array_schema_->dense()) {
init_domain(non_empty_domain);
} else {
non_empty_domain_ = std::move(non_empty_domain);
}
}

FragmentMetadata::~FragmentMetadata() = default;

/* ****************************** */
Expand Down Expand Up @@ -1341,10 +1422,6 @@ URI FragmentMetadata::validity_uri(const std::string& name) const {
encoded_name + "_validity" + constants::file_suffix);
}

const std::string& FragmentMetadata::array_schema_name() {
return array_schema_name_;
}

void FragmentMetadata::load_tile_offsets(
const EncryptionKey& encryption_key, std::vector<std::string>& names) {
// Sort 'names' in ascending order of their index. The
Expand Down Expand Up @@ -1624,7 +1701,7 @@ uint64_t FragmentMetadata::tile_size(
}

uint64_t FragmentMetadata::tile_var_size(
const std::string& name, uint64_t tile_idx) {
const std::string& name, uint64_t tile_idx) const {
auto it = idx_map_.find(name);
assert(it != idx_map_.end());
auto idx = it->second;
Expand Down Expand Up @@ -1844,7 +1921,8 @@ uint64_t FragmentMetadata::get_tile_null_count(
return tile_null_counts_[idx][tile_idx];
}

std::vector<uint8_t>& FragmentMetadata::get_min(const std::string& name) {
const std::vector<uint8_t>& FragmentMetadata::get_min(
const std::string& name) const {
auto it = idx_map_.find(name);
assert(it != idx_map_.end());
auto idx = it->second;
Expand All @@ -1866,7 +1944,8 @@ std::vector<uint8_t>& FragmentMetadata::get_min(const std::string& name) {
return fragment_mins_[idx];
}

std::vector<uint8_t>& FragmentMetadata::get_max(const std::string& name) {
const std::vector<uint8_t>& FragmentMetadata::get_max(
const std::string& name) const {
auto it = idx_map_.find(name);
assert(it != idx_map_.end());
auto idx = it->second;
Expand All @@ -1888,7 +1967,7 @@ std::vector<uint8_t>& FragmentMetadata::get_max(const std::string& name) {
return fragment_maxs_[idx];
}

void* FragmentMetadata::get_sum(const std::string& name) {
const void* FragmentMetadata::get_sum(const std::string& name) const {
auto it = idx_map_.find(name);
assert(it != idx_map_.end());
auto idx = it->second;
Expand Down Expand Up @@ -4616,23 +4695,6 @@ void FragmentMetadata::store_footer(const EncryptionKey&) {
resources_->stats().add_counter("write_frag_meta_footer_size", tile.size());
}

void FragmentMetadata::resize_tile_offsets_vectors(uint64_t size) {
tile_offsets_mtx().resize(size);
tile_offsets().resize(size);
}

void FragmentMetadata::resize_tile_var_offsets_vectors(uint64_t size) {
tile_var_offsets_mtx().resize(size);
tile_var_offsets().resize(size);
}

void FragmentMetadata::resize_tile_var_sizes_vectors(uint64_t size) {
tile_var_sizes().resize(size);
}
void FragmentMetadata::resize_tile_validity_offsets_vectors(uint64_t size) {
tile_validity_offsets().resize(size);
}

void FragmentMetadata::clean_up() {
auto fragment_metadata_uri =
fragment_uri_.join_path(constants::fragment_metadata_filename);
Expand All @@ -4641,10 +4703,6 @@ void FragmentMetadata::clean_up() {
throw_if_not_ok(resources_->vfs().remove_file(fragment_metadata_uri));
}

const shared_ptr<const ArraySchema>& FragmentMetadata::array_schema() const {
return array_schema_;
}

void FragmentMetadata::build_idx_map() {
idx_map_.clear();

Expand All @@ -4669,14 +4727,6 @@ void FragmentMetadata::build_idx_map() {
}
}

void FragmentMetadata::set_schema_name(const std::string& name) {
array_schema_name_ = name;
}

void FragmentMetadata::set_dense(bool dense) {
dense_ = dense;
}

// Explicit template instantiations
template std::vector<std::pair<uint64_t, double>>
FragmentMetadata::compute_overlapping_tile_ids_cov<int8_t>(
Expand Down
Loading
Loading