Skip to content

Commit

Permalink
getLibraries with proof
Browse files Browse the repository at this point in the history
  • Loading branch information
@dungeon-master-666
dungeon-master-666 committed Jan 7, 2024
1 parent 23f8e44 commit 977fdf6
Show file tree
Hide file tree
Showing 6 changed files with 224 additions and 34 deletions.
2 changes: 2 additions & 0 deletions tl/generate/scheme/lite_api.tl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ liteServer.partialBlockProof complete:Bool from:tonNode.blockIdExt to:tonNode.bl
liteServer.configInfo mode:# id:tonNode.blockIdExt state_proof:bytes config_proof:bytes = liteServer.ConfigInfo;
liteServer.validatorStats mode:# id:tonNode.blockIdExt count:int complete:Bool state_proof:bytes data_proof:bytes = liteServer.ValidatorStats;
liteServer.libraryResult result:(vector liteServer.libraryEntry) = liteServer.LibraryResult;
liteServer.libraryResultV2 id:tonNode.blockIdExt result:(vector liteServer.libraryEntry) state_proof:bytes data_proof:bytes = liteServer.LibraryResultV2;
liteServer.shardBlockLink id:tonNode.blockIdExt proof:bytes = liteServer.ShardBlockLink;
liteServer.shardBlockProof masterchain_id:tonNode.blockIdExt links:(vector liteServer.shardBlockLink) = liteServer.ShardBlockProof;

Expand Down Expand Up @@ -83,6 +84,7 @@ liteServer.getConfigAll mode:# id:tonNode.blockIdExt = liteServer.ConfigInfo;
liteServer.getConfigParams mode:# id:tonNode.blockIdExt param_list:(vector int) = liteServer.ConfigInfo;
liteServer.getValidatorStats#091a58bc mode:# id:tonNode.blockIdExt limit:int start_after:mode.0?int256 modified_after:mode.2?int = liteServer.ValidatorStats;
liteServer.getLibraries library_list:(vector int256) = liteServer.LibraryResult;
liteServer.getLibrariesV2 id:tonNode.blockIdExt library_list:(vector int256) = liteServer.LibraryResultV2;
liteServer.getShardBlockProof id:tonNode.blockIdExt = liteServer.ShardBlockProof;

liteServer.queryPrefix = Object;
Expand Down
Binary file modified tl/generate/scheme/lite_api.tlo
View file
Binary file not shown.
169 changes: 135 additions & 34 deletions tonlib/tonlib/TonlibClient.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4494,10 +4494,31 @@ void deep_library_search(std::set<td::Bits256>& set, std::set<vm::Cell::Hash>& v

td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request,
td::Promise<object_ptr<tonlib_api::smc_libraryResult>>&& promise) {
if (request.library_list_.size() > 16) {
promise.set_error(TonlibError::Internal("too many libraries requested, 16 maximum"));
}
if (query_context_.block_id) {
get_libs(query_context_.block_id.value(), request.library_list_, std::move(promise));
} else {
client_.with_last_block([this, promise = std::move(promise), library_list = request.library_list_](td::Result<LastBlockState> r_last_block) mutable {
if (r_last_block.is_error()) {
promise.set_error(r_last_block.move_as_error_prefix(TonlibError::Internal("get last block failed ")));
} else {
this->get_libs(r_last_block.move_as_ok().last_block_id, library_list, std::move(promise));
}
});
}
return td::Status::OK();
}

void TonlibClient::get_libs(ton::BlockIdExt blkid, std::vector<td::Bits256> library_list, td::Promise<object_ptr<tonlib_api::smc_libraryResult>>&& promise) {
sort(library_list.begin(), library_list.end());
library_list.erase(unique(library_list.begin(), library_list.end()), library_list.end());

std::vector<object_ptr<tonlib_api::smc_libraryEntry>> result_entries;
result_entries.reserve(request.library_list_.size());
result_entries.reserve(library_list.size());
std::vector<td::Bits256> not_cached_hashes;
for (auto& library_hash : request.library_list_) {
for (auto& library_hash : library_list) {
if (libraries.key_exists(library_hash)) {
auto library_content = vm::std_boc_serialize(libraries.lookup_ref(library_hash)).move_as_ok().as_slice().str();
result_entries.push_back(tonlib_api::make_object<tonlib_api::smc_libraryEntry>(library_hash, library_content));
Expand All @@ -4508,42 +4529,120 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request,

if (not_cached_hashes.empty()) {
promise.set_value(tonlib_api::make_object<tonlib_api::smc_libraryResult>(std::move(result_entries)));
return td::Status::OK();
return;
}

client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(not_cached_hashes)),
promise.wrap([self=this, result_entries = std::move(result_entries)]
(td::Result<ton::lite_api::object_ptr<ton::lite_api::liteServer_libraryResult>> r_libraries) mutable
{
if (r_libraries.is_error()) {
LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string();
} else {
auto libraries = r_libraries.move_as_ok();
bool updated = false;
for (auto& lr : libraries->result_) {
auto contents = vm::std_boc_deserialize(lr->data_);
if (contents.is_ok() && contents.ok().not_null()) {
if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) {
LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex();
continue;
}
result_entries.push_back(tonlib_api::make_object<tonlib_api::smc_libraryEntry>(lr->hash_, lr->data_.as_slice().str()));
self->libraries.set_ref(lr->hash_, contents.move_as_ok());
updated = true;
LOG(DEBUG) << "registered library " << lr->hash_.to_hex();
} else {
LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex();
}
if (updated) {
self->store_libs_to_disk();
}
client_.send_query(ton::lite_api::liteServer_getLibrariesV2(ton::create_tl_lite_block_id(blkid), std::move(not_cached_hashes)),
promise.wrap([self=this, blkid, result_entries = std::move(result_entries), not_cached_hashes]
(td::Result<ton::lite_api::object_ptr<ton::lite_api::liteServer_libraryResultV2>> r_libraries) mutable
-> td::Result<tonlib_api::object_ptr<tonlib_api::smc_libraryResult>> {
if (r_libraries.is_error()) {
LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string();
return r_libraries.move_as_error();
}

auto libraries = r_libraries.move_as_ok();
auto state = block::check_extract_state_proof(blkid, libraries->state_proof_.as_slice(),
libraries->data_proof_.as_slice());
if (state.is_error()) {
LOG(WARNING) << "cannot check state proof: " << state.move_as_error().to_string();
return state.move_as_error();
}
auto state_root = state.move_as_ok();
auto rconfig = block::ConfigInfo::extract_config(state_root, block::ConfigInfo::needLibraries);
if (rconfig.is_error()) {
LOG(WARNING) << "cannot extract config: " << rconfig.move_as_error().to_string();
return rconfig.move_as_error();
}
auto config = rconfig.move_as_ok();
for (auto& hash : not_cached_hashes) {
auto libres = config->lookup_library(hash);
if (libres.is_null()) {
LOG(WARNING) << "library " << hash.to_hex() << " not found in config";
if (std::any_of(libraries->result_.begin(), libraries->result_.end(),
[&hash](const auto& lib) { return lib->hash_.bits().equals(hash.cbits(), 256); })) {
return TonlibError::Internal("library is included in response but it's not found in proof");
}
continue;
}
return tonlib_api::make_object<tonlib_api::smc_libraryResult>(std::move(result_entries));
}));
return td::Status::OK();

auto lib_it = std::find_if(libraries->result_.begin(), libraries->result_.end(),
[&hash](const auto& lib) { return !lib->hash_.bits().compare(hash.cbits(), 256); });
if (lib_it == libraries->result_.end()) {
return TonlibError::Internal("library is found in proof but not in response");
}
auto& lib = *lib_it;
auto contents = vm::std_boc_deserialize(lib->data_);
if (!contents.is_ok() || contents.ok().is_null()) {
return TonlibError::Internal(PSLICE() << "library hash mismatch " << lib->hash_.to_hex());
}

if (contents.ok()->get_hash().bits().compare(hash.cbits(), 256)) {
return TonlibError::Internal(PSLICE() << "library hash mismatch " << contents.ok()->get_hash().to_hex() << " != " << hash.to_hex());
}

result_entries.push_back(tonlib_api::make_object<tonlib_api::smc_libraryEntry>(lib->hash_, lib->data_.as_slice().str()));
self->libraries.set_ref(lib->hash_, contents.move_as_ok());
LOG(DEBUG) << "registered library " << lib->hash_.to_hex();
}
self->store_libs_to_disk();
sort(result_entries.begin(), result_entries.end());
return tonlib_api::make_object<tonlib_api::smc_libraryResult>(std::move(result_entries));
}));
}

// td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request,
// td::Promise<object_ptr<tonlib_api::smc_libraryResult>>&& promise) {
// std::vector<object_ptr<tonlib_api::smc_libraryEntry>> result_entries;
// result_entries.reserve(request.library_list_.size());
// std::vector<td::Bits256> not_cached_hashes;
// for (auto& library_hash : request.library_list_) {
// if (libraries.key_exists(library_hash)) {
// auto library_content = vm::std_boc_serialize(libraries.lookup_ref(library_hash)).move_as_ok().as_slice().str();
// result_entries.push_back(tonlib_api::make_object<tonlib_api::smc_libraryEntry>(library_hash, library_content));
// } else {
// not_cached_hashes.push_back(library_hash);
// }
// }

// if (not_cached_hashes.empty()) {
// promise.set_value(tonlib_api::make_object<tonlib_api::smc_libraryResult>(std::move(result_entries)));
// return td::Status::OK();
// }

// client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(not_cached_hashes)),
// promise.wrap([self=this, result_entries = std::move(result_entries)]
// (td::Result<ton::lite_api::object_ptr<ton::lite_api::liteServer_libraryResult>> r_libraries) mutable
// {
// if (r_libraries.is_error()) {
// LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string();
// } else {
// auto libraries = r_libraries.move_as_ok();
// bool updated = false;
// for (auto& lr : libraries->result_) {
// auto contents = vm::std_boc_deserialize(lr->data_);
// if (contents.is_ok() && contents.ok().not_null()) {
// if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) {
// LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex();
// continue;
// }
// result_entries.push_back(tonlib_api::make_object<tonlib_api::smc_libraryEntry>(lr->hash_, lr->data_.as_slice().str()));
// self->libraries.set_ref(lr->hash_, contents.move_as_ok());
// updated = true;
// LOG(DEBUG) << "registered library " << lr->hash_.to_hex();
// } else {
// LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex();
// }
// if (updated) {
// self->store_libs_to_disk();
// }
// }
// }
// return tonlib_api::make_object<tonlib_api::smc_libraryResult>(std::move(result_entries));
// }));
// return td::Status::OK();
// }

td::Status TonlibClient::do_request(const tonlib_api::smc_getLibrariesExt& request,
td::Promise<object_ptr<tonlib_api::smc_libraryResultExt>>&& promise) {
std::set<td::Bits256> request_libs;
Expand Down Expand Up @@ -5799,6 +5898,7 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShardBlockProof&
}

void TonlibClient::load_libs_from_disk() {
std::lock_guard<std::mutex> guard(libs_mutex_);
LOG(DEBUG) << "loading libraries from disk cache";
auto r_data = kv_->get("tonlib.libcache");
if (r_data.is_error()) {
Expand All @@ -5810,14 +5910,15 @@ void TonlibClient::load_libs_from_disk() {
}
libraries = vm::Dictionary(vm::load_cell_slice(vm::CellBuilder().append_cellslice(vm::load_cell_slice(
r_dict.move_as_ok())).finalize()), 256);
// int n = 0; for (auto&& lr : libraries) n++;

LOG(DEBUG) << "loaded libraries from disk cache";
}

void TonlibClient::store_libs_to_disk() { // NB: Dictionary.get_root_cell does not compute_root, and it is protected
std::lock_guard<std::mutex> guard(libs_mutex_);
kv_->set("tonlib.libcache", vm::std_boc_serialize(vm::CellBuilder().append_cellslice(libraries.get_root())
.finalize()).move_as_ok().as_slice());
// int n = 0; for (auto&& lr : libraries) n++;

LOG(DEBUG) << "stored libraries to disk cache";
}

Expand Down
2 changes: 2 additions & 0 deletions tonlib/tonlib/TonlibClient.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,7 @@ class TonlibClient : public td::actor::Actor {

td::Status do_request(const tonlib_api::smc_getLibraries& request,
td::Promise<object_ptr<tonlib_api::smc_libraryResult>>&& promise);
void get_libs(ton::BlockIdExt blkid, std::vector<td::Bits256> library_list_, td::Promise<object_ptr<tonlib_api::smc_libraryResult>>&& promise);

td::Status do_request(const tonlib_api::smc_getLibrariesExt& request,
td::Promise<object_ptr<tonlib_api::smc_libraryResultExt>>&& promise);
Expand Down Expand Up @@ -401,6 +402,7 @@ class TonlibClient : public td::actor::Actor {

void proxy_request(td::int64 query_id, std::string data);

std::mutex libs_mutex_;
void load_libs_from_disk();
void store_libs_to_disk();

Expand Down
83 changes: 83 additions & 0 deletions validator/impl/liteserver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,9 @@ void LiteQuery::start_up() {
[&](lite_api::liteServer_getLibraries& q) {
this->perform_getLibraries(q.library_list_);
},
[&](lite_api::liteServer_getLibrariesV2& q) {
this->perform_getLibrariesV2(ton::create_block_id(q.id_), q.library_list_);
},
[&](lite_api::liteServer_getShardBlockProof& q) {
this->perform_getShardBlockProof(create_block_id(q.id_));
},
Expand Down Expand Up @@ -918,6 +921,86 @@ void LiteQuery::continue_getLibraries(Ref<ton::validator::MasterchainState> mc_s
finish_query(std::move(b));
}

void LiteQuery::perform_getLibrariesV2(BlockIdExt blkid, std::vector<td::Bits256> library_list) {
LOG(INFO) << "started a getLibrariesV2(<list of " << library_list.size() << " parameters>) liteserver query";
if (library_list.size() > 16) {
LOG(INFO) << "too many libraries requested, returning only first 16";
library_list.resize(16);
}
sort( library_list.begin(), library_list.end() );
library_list.erase( unique( library_list.begin(), library_list.end() ), library_list.end() );

set_continuation([this, library_list]() -> void { continue_getLibrariesV2(library_list); });
request_mc_block_data_state(blkid);
}

void LiteQuery::continue_getLibrariesV2(std::vector<td::Bits256> library_list) {
LOG(INFO) << "obtained masterchain block = " << base_blk_id_.to_str();
CHECK(mc_state_.not_null());

Ref<vm::Cell> state_proof, data_proof;
if (!make_mc_state_root_proof(state_proof)) {
return;
}

vm::MerkleProofBuilder pb{state_->root_cell()};

auto rconfig = block::ConfigInfo::extract_config(pb.root(), block::ConfigInfo::needLibraries);
if (rconfig.is_error()) {
fatal_error("cannot extract library list block configuration from masterchain state");
return;
}
auto config = rconfig.move_as_ok();

if (false) {
std::ostringstream os;
vm::load_cell_slice(config->get_libraries_root()).print_rec(os);
LOG(INFO) << "\n" << os.str();

auto lib_dict = std::make_unique<vm::Dictionary>(config->get_libraries_root(), 256);
for (auto k: *lib_dict) {
std::ostringstream oss;
k.second->print_rec(oss);
LOG(INFO) << "library " << k.first.to_hex(256) << ": \n" << oss.str();
}
}

std::vector<ton::tl_object_ptr<ton::lite_api::liteServer_libraryEntry>> result;
for (const auto& hash : library_list) {
LOG(INFO) << "looking for library " << hash.to_hex();
auto libres = config->lookup_library(hash);
if (libres.is_null()) {
LOG(INFO) << "library lookup result is null";
continue;
}
auto data = vm::std_boc_serialize(libres);
if (data.is_error()) {
LOG(WARNING) << "library serialization failed: " << data.move_as_error().to_string();
continue;
}
result.push_back(ton::create_tl_object<ton::lite_api::liteServer_libraryEntry>(hash, data.move_as_ok()));
}

if (!pb.extract_proof_to(data_proof)) {
fatal_error("error while constructing Merkle proof for library list");
return;
}
auto state_proof_boc = vm::std_boc_serialize(std::move(state_proof));
if (state_proof_boc.is_error()) {
fatal_error(state_proof_boc.move_as_error());
return;
}
auto data_proof_boc = vm::std_boc_serialize(std::move(data_proof));
if (data_proof_boc.is_error()) {
fatal_error(data_proof_boc.move_as_error());
return;
}

auto b = ton::create_serialize_tl_object<ton::lite_api::liteServer_libraryResultV2>(ton::create_tl_lite_block_id(base_blk_id_), std::move(result),
state_proof_boc.move_as_ok(), data_proof_boc.move_as_ok());
finish_query(std::move(b));
}

void LiteQuery::perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt) {
LOG(INFO) << "started a getOneTransaction(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << ","
<< lt << ") liteserver query";
Expand Down
2 changes: 2 additions & 0 deletions validator/impl/liteserver.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,8 @@ class LiteQuery : public td::actor::Actor {
UnixTime gen_utime, LogicalTime gen_lt);
void perform_getLibraries(std::vector<td::Bits256> library_list);
void continue_getLibraries(Ref<MasterchainState> mc_state, BlockIdExt blkid, std::vector<td::Bits256> library_list);
void perform_getLibrariesV2(BlockIdExt blkid, std::vector<td::Bits256> library_list);
void continue_getLibrariesV2(std::vector<td::Bits256> library_list);
void perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt);
void continue_getOneTransaction();
void perform_getTransactions(WorkchainId workchain, StdSmcAddress addr, LogicalTime lt, Bits256 hash, unsigned count);
Expand Down

0 comments on commit 977fdf6

Please sign in to comment.