From 977fdf6359c1fd1ede13732214cbf82286fcdc9b Mon Sep 17 00:00:00 2001 From: ms Date: Sun, 7 Jan 2024 18:50:41 +0100 Subject: [PATCH] getLibraries with proof --- tl/generate/scheme/lite_api.tl | 2 + tl/generate/scheme/lite_api.tlo | Bin 14148 -> 14580 bytes tonlib/tonlib/TonlibClient.cpp | 169 +++++++++++++++++++++++++------- tonlib/tonlib/TonlibClient.h | 2 + validator/impl/liteserver.cpp | 83 ++++++++++++++++ validator/impl/liteserver.hpp | 2 + 6 files changed, 224 insertions(+), 34 deletions(-) diff --git a/tl/generate/scheme/lite_api.tl b/tl/generate/scheme/lite_api.tl index ce4d27316..03a1f2271 100644 --- a/tl/generate/scheme/lite_api.tl +++ b/tl/generate/scheme/lite_api.tl @@ -53,6 +53,7 @@ liteServer.partialBlockProof complete:Bool from:tonNode.blockIdExt to:tonNode.bl liteServer.configInfo mode:# id:tonNode.blockIdExt state_proof:bytes config_proof:bytes = liteServer.ConfigInfo; liteServer.validatorStats mode:# id:tonNode.blockIdExt count:int complete:Bool state_proof:bytes data_proof:bytes = liteServer.ValidatorStats; liteServer.libraryResult result:(vector liteServer.libraryEntry) = liteServer.LibraryResult; +liteServer.libraryResultV2 id:tonNode.blockIdExt result:(vector liteServer.libraryEntry) state_proof:bytes data_proof:bytes = liteServer.LibraryResultV2; liteServer.shardBlockLink id:tonNode.blockIdExt proof:bytes = liteServer.ShardBlockLink; liteServer.shardBlockProof masterchain_id:tonNode.blockIdExt links:(vector liteServer.shardBlockLink) = liteServer.ShardBlockProof; @@ -83,6 +84,7 @@ liteServer.getConfigAll mode:# id:tonNode.blockIdExt = liteServer.ConfigInfo; liteServer.getConfigParams mode:# id:tonNode.blockIdExt param_list:(vector int) = liteServer.ConfigInfo; liteServer.getValidatorStats#091a58bc mode:# id:tonNode.blockIdExt limit:int start_after:mode.0?int256 modified_after:mode.2?int = liteServer.ValidatorStats; liteServer.getLibraries library_list:(vector int256) = liteServer.LibraryResult; +liteServer.getLibrariesV2 id:tonNode.blockIdExt library_list:(vector int256) = liteServer.LibraryResultV2; liteServer.getShardBlockProof id:tonNode.blockIdExt = liteServer.ShardBlockProof; liteServer.queryPrefix = Object; diff --git a/tl/generate/scheme/lite_api.tlo b/tl/generate/scheme/lite_api.tlo index da64ac53bcac03fb1d2eb1ca4fe3940d7e56c29b..7108945f812c5f994397e0dbf36b10aa26607784 100644 GIT binary patch delta 156 zcmX?-_oa~c(QJJy1}HGv$ScgsDt&Selhow5ED}s%MibXdZdPEmU|}@g9LcH950&GU zkYEopVgM$qqVD-7Pu?xB%8kFjD{k delta 51 zcmexTcqEVa(QJJy1}HGx$ScgcIfB)Kh0$p9L{4@7%~K=;_!*@)Kh@aJve`jz3l{)= C2@cT! diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 0b85cb55f..173551c96 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -4494,10 +4494,31 @@ void deep_library_search(std::set& set, std::set& v td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request, td::Promise>&& promise) { + if (request.library_list_.size() > 16) { + promise.set_error(TonlibError::Internal("too many libraries requested, 16 maximum")); + } + if (query_context_.block_id) { + get_libs(query_context_.block_id.value(), request.library_list_, std::move(promise)); + } else { + client_.with_last_block([this, promise = std::move(promise), library_list = request.library_list_](td::Result r_last_block) mutable { + if (r_last_block.is_error()) { + promise.set_error(r_last_block.move_as_error_prefix(TonlibError::Internal("get last block failed "))); + } else { + this->get_libs(r_last_block.move_as_ok().last_block_id, library_list, std::move(promise)); + } + }); + } + return td::Status::OK(); +} + +void TonlibClient::get_libs(ton::BlockIdExt blkid, std::vector library_list, td::Promise>&& promise) { + sort(library_list.begin(), library_list.end()); + library_list.erase(unique(library_list.begin(), library_list.end()), library_list.end()); + std::vector> result_entries; - result_entries.reserve(request.library_list_.size()); + result_entries.reserve(library_list.size()); std::vector not_cached_hashes; - for (auto& library_hash : request.library_list_) { + for (auto& library_hash : library_list) { if (libraries.key_exists(library_hash)) { auto library_content = vm::std_boc_serialize(libraries.lookup_ref(library_hash)).move_as_ok().as_slice().str(); result_entries.push_back(tonlib_api::make_object(library_hash, library_content)); @@ -4508,42 +4529,120 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request, if (not_cached_hashes.empty()) { promise.set_value(tonlib_api::make_object(std::move(result_entries))); - return td::Status::OK(); + return; } - client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(not_cached_hashes)), - promise.wrap([self=this, result_entries = std::move(result_entries)] - (td::Result> r_libraries) mutable - { - if (r_libraries.is_error()) { - LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); - } else { - auto libraries = r_libraries.move_as_ok(); - bool updated = false; - for (auto& lr : libraries->result_) { - auto contents = vm::std_boc_deserialize(lr->data_); - if (contents.is_ok() && contents.ok().not_null()) { - if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { - LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); - continue; - } - result_entries.push_back(tonlib_api::make_object(lr->hash_, lr->data_.as_slice().str())); - self->libraries.set_ref(lr->hash_, contents.move_as_ok()); - updated = true; - LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); - } else { - LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); - } - if (updated) { - self->store_libs_to_disk(); - } + client_.send_query(ton::lite_api::liteServer_getLibrariesV2(ton::create_tl_lite_block_id(blkid), std::move(not_cached_hashes)), + promise.wrap([self=this, blkid, result_entries = std::move(result_entries), not_cached_hashes] + (td::Result> r_libraries) mutable + -> td::Result> { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); + return r_libraries.move_as_error(); + } + + auto libraries = r_libraries.move_as_ok(); + auto state = block::check_extract_state_proof(blkid, libraries->state_proof_.as_slice(), + libraries->data_proof_.as_slice()); + if (state.is_error()) { + LOG(WARNING) << "cannot check state proof: " << state.move_as_error().to_string(); + return state.move_as_error(); + } + auto state_root = state.move_as_ok(); + auto rconfig = block::ConfigInfo::extract_config(state_root, block::ConfigInfo::needLibraries); + if (rconfig.is_error()) { + LOG(WARNING) << "cannot extract config: " << rconfig.move_as_error().to_string(); + return rconfig.move_as_error(); + } + auto config = rconfig.move_as_ok(); + for (auto& hash : not_cached_hashes) { + auto libres = config->lookup_library(hash); + if (libres.is_null()) { + LOG(WARNING) << "library " << hash.to_hex() << " not found in config"; + if (std::any_of(libraries->result_.begin(), libraries->result_.end(), + [&hash](const auto& lib) { return lib->hash_.bits().equals(hash.cbits(), 256); })) { + return TonlibError::Internal("library is included in response but it's not found in proof"); } + continue; } - return tonlib_api::make_object(std::move(result_entries)); - })); - return td::Status::OK(); + + auto lib_it = std::find_if(libraries->result_.begin(), libraries->result_.end(), + [&hash](const auto& lib) { return !lib->hash_.bits().compare(hash.cbits(), 256); }); + if (lib_it == libraries->result_.end()) { + return TonlibError::Internal("library is found in proof but not in response"); + } + auto& lib = *lib_it; + auto contents = vm::std_boc_deserialize(lib->data_); + if (!contents.is_ok() || contents.ok().is_null()) { + return TonlibError::Internal(PSLICE() << "library hash mismatch " << lib->hash_.to_hex()); + } + + if (contents.ok()->get_hash().bits().compare(hash.cbits(), 256)) { + return TonlibError::Internal(PSLICE() << "library hash mismatch " << contents.ok()->get_hash().to_hex() << " != " << hash.to_hex()); + } + + result_entries.push_back(tonlib_api::make_object(lib->hash_, lib->data_.as_slice().str())); + self->libraries.set_ref(lib->hash_, contents.move_as_ok()); + LOG(DEBUG) << "registered library " << lib->hash_.to_hex(); + } + self->store_libs_to_disk(); + sort(result_entries.begin(), result_entries.end()); + return tonlib_api::make_object(std::move(result_entries)); + })); } +// td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request, +// td::Promise>&& promise) { +// std::vector> result_entries; +// result_entries.reserve(request.library_list_.size()); +// std::vector not_cached_hashes; +// for (auto& library_hash : request.library_list_) { +// if (libraries.key_exists(library_hash)) { +// auto library_content = vm::std_boc_serialize(libraries.lookup_ref(library_hash)).move_as_ok().as_slice().str(); +// result_entries.push_back(tonlib_api::make_object(library_hash, library_content)); +// } else { +// not_cached_hashes.push_back(library_hash); +// } +// } + +// if (not_cached_hashes.empty()) { +// promise.set_value(tonlib_api::make_object(std::move(result_entries))); +// return td::Status::OK(); +// } + +// client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(not_cached_hashes)), +// promise.wrap([self=this, result_entries = std::move(result_entries)] +// (td::Result> r_libraries) mutable +// { +// if (r_libraries.is_error()) { +// LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); +// } else { +// auto libraries = r_libraries.move_as_ok(); +// bool updated = false; +// for (auto& lr : libraries->result_) { +// auto contents = vm::std_boc_deserialize(lr->data_); +// if (contents.is_ok() && contents.ok().not_null()) { +// if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { +// LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); +// continue; +// } +// result_entries.push_back(tonlib_api::make_object(lr->hash_, lr->data_.as_slice().str())); +// self->libraries.set_ref(lr->hash_, contents.move_as_ok()); +// updated = true; +// LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); +// } else { +// LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); +// } +// if (updated) { +// self->store_libs_to_disk(); +// } +// } +// } +// return tonlib_api::make_object(std::move(result_entries)); +// })); +// return td::Status::OK(); +// } + td::Status TonlibClient::do_request(const tonlib_api::smc_getLibrariesExt& request, td::Promise>&& promise) { std::set request_libs; @@ -5799,6 +5898,7 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShardBlockProof& } void TonlibClient::load_libs_from_disk() { + std::lock_guard guard(libs_mutex_); LOG(DEBUG) << "loading libraries from disk cache"; auto r_data = kv_->get("tonlib.libcache"); if (r_data.is_error()) { @@ -5810,14 +5910,15 @@ void TonlibClient::load_libs_from_disk() { } libraries = vm::Dictionary(vm::load_cell_slice(vm::CellBuilder().append_cellslice(vm::load_cell_slice( r_dict.move_as_ok())).finalize()), 256); - // int n = 0; for (auto&& lr : libraries) n++; + LOG(DEBUG) << "loaded libraries from disk cache"; } void TonlibClient::store_libs_to_disk() { // NB: Dictionary.get_root_cell does not compute_root, and it is protected + std::lock_guard guard(libs_mutex_); kv_->set("tonlib.libcache", vm::std_boc_serialize(vm::CellBuilder().append_cellslice(libraries.get_root()) .finalize()).move_as_ok().as_slice()); - // int n = 0; for (auto&& lr : libraries) n++; + LOG(DEBUG) << "stored libraries to disk cache"; } diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index f4651b9d1..13a75152d 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -330,6 +330,7 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(const tonlib_api::smc_getLibraries& request, td::Promise>&& promise); + void get_libs(ton::BlockIdExt blkid, std::vector library_list_, td::Promise>&& promise); td::Status do_request(const tonlib_api::smc_getLibrariesExt& request, td::Promise>&& promise); @@ -401,6 +402,7 @@ class TonlibClient : public td::actor::Actor { void proxy_request(td::int64 query_id, std::string data); + std::mutex libs_mutex_; void load_libs_from_disk(); void store_libs_to_disk(); diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 53bbb1ea8..e7a957d39 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -210,6 +210,9 @@ void LiteQuery::start_up() { [&](lite_api::liteServer_getLibraries& q) { this->perform_getLibraries(q.library_list_); }, + [&](lite_api::liteServer_getLibrariesV2& q) { + this->perform_getLibrariesV2(ton::create_block_id(q.id_), q.library_list_); + }, [&](lite_api::liteServer_getShardBlockProof& q) { this->perform_getShardBlockProof(create_block_id(q.id_)); }, @@ -918,6 +921,86 @@ void LiteQuery::continue_getLibraries(Ref mc_s finish_query(std::move(b)); } +void LiteQuery::perform_getLibrariesV2(BlockIdExt blkid, std::vector library_list) { + LOG(INFO) << "started a getLibrariesV2() liteserver query"; + if (library_list.size() > 16) { + LOG(INFO) << "too many libraries requested, returning only first 16"; + library_list.resize(16); + } + sort( library_list.begin(), library_list.end() ); + library_list.erase( unique( library_list.begin(), library_list.end() ), library_list.end() ); + + set_continuation([this, library_list]() -> void { continue_getLibrariesV2(library_list); }); + request_mc_block_data_state(blkid); +} + +void LiteQuery::continue_getLibrariesV2(std::vector library_list) { + LOG(INFO) << "obtained masterchain block = " << base_blk_id_.to_str(); + CHECK(mc_state_.not_null()); + + Ref state_proof, data_proof; + if (!make_mc_state_root_proof(state_proof)) { + return; + } + + vm::MerkleProofBuilder pb{state_->root_cell()}; + + auto rconfig = block::ConfigInfo::extract_config(pb.root(), block::ConfigInfo::needLibraries); + if (rconfig.is_error()) { + fatal_error("cannot extract library list block configuration from masterchain state"); + return; + } + auto config = rconfig.move_as_ok(); + + if (false) { + std::ostringstream os; + vm::load_cell_slice(config->get_libraries_root()).print_rec(os); + LOG(INFO) << "\n" << os.str(); + + auto lib_dict = std::make_unique(config->get_libraries_root(), 256); + for (auto k: *lib_dict) { + std::ostringstream oss; + k.second->print_rec(oss); + LOG(INFO) << "library " << k.first.to_hex(256) << ": \n" << oss.str(); + } + } + + std::vector> result; + for (const auto& hash : library_list) { + LOG(INFO) << "looking for library " << hash.to_hex(); + auto libres = config->lookup_library(hash); + if (libres.is_null()) { + LOG(INFO) << "library lookup result is null"; + continue; + } + auto data = vm::std_boc_serialize(libres); + if (data.is_error()) { + LOG(WARNING) << "library serialization failed: " << data.move_as_error().to_string(); + continue; + } + result.push_back(ton::create_tl_object(hash, data.move_as_ok())); + } + + if (!pb.extract_proof_to(data_proof)) { + fatal_error("error while constructing Merkle proof for library list"); + return; + } + auto state_proof_boc = vm::std_boc_serialize(std::move(state_proof)); + if (state_proof_boc.is_error()) { + fatal_error(state_proof_boc.move_as_error()); + return; + } + auto data_proof_boc = vm::std_boc_serialize(std::move(data_proof)); + if (data_proof_boc.is_error()) { + fatal_error(data_proof_boc.move_as_error()); + return; + } + + auto b = ton::create_serialize_tl_object(ton::create_tl_lite_block_id(base_blk_id_), std::move(result), + state_proof_boc.move_as_ok(), data_proof_boc.move_as_ok()); + finish_query(std::move(b)); +} + void LiteQuery::perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt) { LOG(INFO) << "started a getOneTransaction(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << "," << lt << ") liteserver query"; diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index 2707fdfed..ab1418b08 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -117,6 +117,8 @@ class LiteQuery : public td::actor::Actor { UnixTime gen_utime, LogicalTime gen_lt); void perform_getLibraries(std::vector library_list); void continue_getLibraries(Ref mc_state, BlockIdExt blkid, std::vector library_list); + void perform_getLibrariesV2(BlockIdExt blkid, std::vector library_list); + void continue_getLibrariesV2(std::vector library_list); void perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt); void continue_getOneTransaction(); void perform_getTransactions(WorkchainId workchain, StdSmcAddress addr, LogicalTime lt, Bits256 hash, unsigned count);