celldb: add test for load nonexisting cell, test thread safeness of C…

…ellUsageTree, fixes
ton-blockchain · Feb 25, 2025 · 597f3b7 · 597f3b7
1 parent c863c42
commit 597f3b7
Show file tree

Hide file tree

Showing 7 changed files with 130 additions and 61 deletions.
diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp
@@ -50,6 +50,7 @@
 #include <set>
 #include <map>
 #include <thread>
+#include <barrier>
 
 #include <openssl/sha.h>
 
@@ -1312,7 +1313,8 @@ void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) {
       // V2 - one thread
       run({.async_executor = executor,
            .kv_options = kv_options,
-           .options = DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}});
+           .options =
+               DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}});
 
       // InMemory
       for (auto use_arena : {false, true}) {
@@ -1353,6 +1355,8 @@ DynamicBagOfCellsDb::Stats test_dynamic_boc(BocOptions options) {
     if (rnd() % 10 == 0) {
       reload_db();
     }
+    db.dboc->load_cell(vm::CellHash{}.as_slice()).ensure_error();
+
     db.reset_loader();
     Ref<Cell> old_root;
     if (!old_root_hash.empty()) {
@@ -2901,6 +2905,35 @@ TEST(TonDb, BocRespectsUsageCell) {
   ASSERT_STREQ(serialization, serialization_of_virtualized_cell);
 }
 
+TEST(UsageTree, ThreadSafe) {
+  size_t test_n = 100;
+  td::Random::Xorshift128plus rnd(123);
+  for (size_t test_i = 0; test_i < test_n; test_i++) {
+    auto cell = vm::gen_random_cell(rnd.fast(2, 100), rnd, false);
+    auto usage_tree = std::make_shared<vm::CellUsageTree>();
+    auto usage_cell = vm::UsageCell::create(cell, usage_tree->root_ptr());
+    std::ptrdiff_t threads_n = 1; // TODO: when CellUsageTree is thread safe, change it to 4
+    auto barrier = std::barrier{threads_n};
+    std::vector<std::thread> threads;
+    std::vector<vm::CellExplorer::Exploration> explorations(threads_n);
+    for (std::ptrdiff_t i = 0; i < threads_n; i++) {
+      threads.emplace_back([&, i = i]() {
+        barrier.arrive_and_wait();
+        explorations[i] = vm::CellExplorer::random_explore(usage_cell, rnd);
+      });
+    }
+    for (auto &thread : threads) {
+      thread.join();
+    }
+    auto proof = vm::MerkleProof::generate(cell, usage_tree.get());
+    auto virtualized_proof = vm::MerkleProof::virtualize(proof, 1);
+    for (auto &exploration : explorations) {
+      auto new_exploration = vm::CellExplorer::explore(virtualized_proof, exploration.ops);
+      ASSERT_EQ(exploration.log, new_exploration.log);
+    }
+  }
+}
+
 /*
 vm::DynamicBagOfCellsDb::Stats test_dynamic_boc_respects_usage_cell(vm::BocOptions options) {
   td::Random::Xorshift128plus rnd(options.seed);

diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp
@@ -107,15 +107,21 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat
   td::Result<Ref<Cell>> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override {
     return get_cell_info_lazy(level_mask, hash, depth).cell;
   }
-  td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all() const override {
+  td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all(size_t max_count) const override {
     std::vector<std::pair<std::string, std::string>> result;
     auto s = loader_->key_value_reader().for_each_in_range("desc", "desd",
                                                            [&](const td::Slice &key, const td::Slice &value) {
+                                                             if (result.size() >= max_count) {
+                                                               return td::Status::Error("COUNT_LIMIT");
+                                                             }
                                                              if (td::begins_with(key, "desc") && key.size() != 32) {
                                                                result.emplace_back(key.str(), value.str());
                                                              }
                                                              return td::Status::OK();
                                                            });
+    if (s.message() == "COUNT_LIMIT") {
+      s = td::Status::OK();
+    }
     TRY_STATUS(std::move(s));
     return result;
   }

diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h
@@ -51,7 +51,7 @@ class DynamicBagOfCellsDb {
  public:
   virtual ~DynamicBagOfCellsDb() = default;
 
-  virtual td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all() const = 0;
+  virtual td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all(size_t max_count) const = 0;
   virtual td::Result<td::KeyValue::GetStatus> meta_get(td::Slice key, std::string &value) = 0;
   virtual td::Status meta_set(td::Slice key, td::Slice value) = 0;
   virtual td::Status meta_erase(td::Slice key) = 0;

diff --git a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp
@@ -765,16 +765,22 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb {
     }
   }
 
-  td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all() const override {
+  td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all(size_t max_count) const override {
     CHECK(meta_db_fixup_.empty());
     std::vector<std::pair<std::string, std::string>> result;
     auto s = cell_db_reader_->key_value_reader().for_each_in_range(
         "desc", "desd", [&](const td::Slice &key, const td::Slice &value) {
+           if (result.size() >= max_count) {
+             return td::Status::Error("COUNT_LIMIT");
+           }
           if (td::begins_with(key, "desc") && key.size() != 32) {
             result.emplace_back(key.str(), value.str());
           }
           return td::Status::OK();
         });
+    if (s.message() == "COUNT_LIMIT") {
+      s = td::Status::OK();
+    }
     TRY_STATUS(std::move(s));
     return result;
   }

diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp
@@ -774,8 +774,15 @@ class MetaStorage {
       CHECK(p.first.size() != CellTraits::hash_bytes);
     }
   }
-  std::vector<std::pair<std::string, std::string>> meta_get_all() const {
-    return td::transform(meta_, [](const auto &p) {  return std::make_pair(p.first, p.second); });
+  std::vector<std::pair<std::string, std::string>> meta_get_all(size_t max_count) const {
+    std::vector<std::pair<std::string, std::string>> res;
+    for (const auto &[k, v] : meta_) {
+      if (res.size() >= max_count) {
+        break;
+      }
+      res.emplace_back(k, v);
+    }
+    return res;
   }
   KeyValue::GetStatus meta_get(td::Slice key, std::string &value) const {
     auto lock = local_access_.lock();
@@ -814,8 +821,8 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb {
       : storage_(std::move(storage)), meta_storage_(std::move(meta_storage)) {
   }
 
-  td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all() const override {
-    return meta_storage_->meta_get_all();
+  td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all(size_t max_count) const override {
+    return meta_storage_->meta_get_all(max_count);
   }
   td::Result<KeyValue::GetStatus> meta_get(td::Slice key, std::string &value) override {
     CHECK(key.size() != CellTraits::hash_bytes);

diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp
@@ -111,6 +111,66 @@ struct MergeOperatorAddCellRefcnt : public rocksdb::MergeOperator {
   }
 };
 
+void CellDbIn::validate_meta() {
+  LOG(INFO) << "Validating metadata\n";
+  size_t max_meta_keys_loaded = opts_->get_celldb_in_memory() ? std::numeric_limits<std::size_t>::max() : 10000;
+  auto meta = boc_->meta_get_all(max_meta_keys_loaded).move_as_ok();
+  bool partial_check = meta.size() == max_meta_keys_loaded;
+  if (partial_check) {
+    LOG(ERROR) << "Too much metadata in the database, do only partial check";
+  }
+  size_t missing_roots = 0;
+  size_t unknown_roots = 0;
+  std::set<vm::CellHash> root_hashes;
+  for (auto [k, v] : meta) {
+    if (k == "desczero") {
+      continue;
+    }
+    auto obj = fetch_tl_object<ton_api::db_celldb_value>(td::BufferSlice{v}, true);
+    obj.ensure();
+    auto entry = DbEntry{obj.move_as_ok()};
+    root_hashes.insert(vm::CellHash::from_slice(entry.root_hash.as_slice()));
+    auto cell = boc_->load_cell(entry.root_hash.as_slice());
+    missing_roots += cell.is_error();
+    LOG_IF(ERROR, cell.is_error()) << "Cannot load root from meta: " << entry.block_id.to_str() << " " << cell.error();
+  }
+
+  // load_known_roots is only supported by InMemory database, so it is ok to check all known roots here
+  auto known_roots = boc_->load_known_roots().move_as_ok();
+  for (auto& root : known_roots) {
+    block::gen::ShardStateUnsplit::Record info;
+    block::gen::OutMsgQueueInfo::Record qinfo;
+    block::ShardId shard;
+    if (!(tlb::unpack_cell(root, info) && shard.deserialize(info.shard_id.write()) &&
+          tlb::unpack_cell(info.out_msg_queue_info, qinfo))) {
+      LOG(FATAL) << "cannot create ShardDescr from a root in celldb";
+    }
+    if (!partial_check && !root_hashes.contains(root->get_hash())) {
+      unknown_roots++;
+      LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no;
+      constexpr bool delete_unknown_roots = false;
+      if (delete_unknown_roots) {
+        vm::CellStorer stor{*cell_db_};
+        cell_db_->begin_write_batch().ensure();
+        boc_->dec(root);
+        boc_->commit(stor).ensure();
+        cell_db_->commit_write_batch().ensure();
+        if (!opts_->get_celldb_in_memory()) {
+          boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
+        }
+        LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no << " REMOVED";
+      }
+    }
+  }
+
+  LOG_IF(ERROR, missing_roots != 0) << "Missing root hashes: " << missing_roots;
+  LOG_IF(ERROR, unknown_roots != 0) << "Unknown roots: " << unknown_roots;
+
+  LOG_IF(FATAL, missing_roots != 0) << "Missing root hashes: " << missing_roots;
+  LOG_IF(FATAL, unknown_roots != 0) << "Unknown roots: " << unknown_roots;
+  LOG(INFO) << "Validating metadata: OK\n";
+}
+
 void CellDbIn::start_up() {
   on_load_callback_ = [actor = std::make_shared<td::actor::ActorOwn<MigrationProxy>>(
                            td::actor::create_actor<MigrationProxy>("celldbmigration", actor_id(this))),
@@ -141,11 +201,11 @@ void CellDbIn::start_up() {
   std::optional<vm::DynamicBagOfCellsDb::CreateV2Options> boc_v2_options;
 
   if (opts_->get_celldb_v2()) {
-    boc_v2_options =
-        vm::DynamicBagOfCellsDb::CreateV2Options{.extra_threads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u),
-                                                 .executor = {},
-                                                 .cache_ttl_max = 2000,
-                                                 .cache_size_max = 1000000};
+    boc_v2_options = vm::DynamicBagOfCellsDb::CreateV2Options{
+        .extra_threads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u),
+        .executor = {},
+        .cache_ttl_max = 2000,
+        .cache_size_max = 1000000};
     size_t min_rocksdb_cache = std::max(size_t{1} << 30, boc_v2_options->cache_size_max * 5000);
     if (!o_celldb_cache_size || o_celldb_cache_size.value() < min_rocksdb_cache) {
       LOG(WARNING) << "Increase CellDb block cache size to " << td::format::as_size(min_rocksdb_cache) << " from "
@@ -208,54 +268,7 @@ void CellDbIn::start_up() {
     boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
   }
 
-  auto meta = boc_->meta_get_all().move_as_ok();
-  size_t missing_roots = 0;
-  size_t unknown_roots = 0;
-  std::set<vm::CellHash> root_hashes;
-  for (auto [k, v] : meta) {
-    if (k == "desczero") {
-      continue;
-    }
-    auto obj = fetch_tl_object<ton_api::db_celldb_value>(td::BufferSlice{v}, true);
-    obj.ensure();
-    auto entry = DbEntry{obj.move_as_ok()};
-    root_hashes.insert(vm::CellHash::from_slice(entry.root_hash.as_slice()));
-    auto cell = boc_->load_cell(entry.root_hash.as_slice());
-    missing_roots += cell.is_error();
-    LOG_IF(ERROR, cell.is_error()) << "Cannot load root from meta: " << entry.block_id.to_str() << " " << cell.error();
-  }
-  auto known_roots = boc_->load_known_roots().move_as_ok();
-  for (auto& root : known_roots) {
-    block::gen::ShardStateUnsplit::Record info;
-    block::gen::OutMsgQueueInfo::Record qinfo;
-    block::ShardId shard;
-    if (!(tlb::unpack_cell(root, info) && shard.deserialize(info.shard_id.write()) &&
-          tlb::unpack_cell(info.out_msg_queue_info, qinfo))) {
-      LOG(FATAL) << "cannot create ShardDescr from a root in celldb";
-    }
-    if (!root_hashes.contains(root->get_hash())) {
-      unknown_roots++;
-      LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no;
-      constexpr bool delete_unknown_roots = false;
-      if (delete_unknown_roots) {
-        vm::CellStorer stor{*cell_db_};
-        cell_db_->begin_write_batch().ensure();
-        boc_->dec(root);
-        boc_->commit(stor).ensure();
-        cell_db_->commit_write_batch().ensure();
-        if (!opts_->get_celldb_in_memory()) {
-          boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
-        }
-        LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no << " REMOVED";
-      }
-    }
-  }
-
-  LOG_IF(ERROR, missing_roots != 1) << "Missing root hashes: " << missing_roots;
-  LOG_IF(ERROR, unknown_roots != 0) << "Unknown roots: " << unknown_roots;
-
-  LOG_IF(FATAL, missing_roots > 1) << "Missing root hashes: " << missing_roots;
-  LOG_IF(FATAL, unknown_roots != 0) << "Unknown roots: " << unknown_roots;
+  validate_meta();
 
   alarm_timestamp() = td::Timestamp::in(10.0);
 
@@ -267,6 +280,9 @@ void CellDbIn::start_up() {
     set_block(empty, std::move(e));
     boc_->commit(stor);
     cell_db_->commit_write_batch().ensure();
+    if (!opts_->get_celldb_in_memory()) {
+      boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
+    }
   }
 
   if (opts_->get_celldb_v2() || opts_->get_celldb_in_memory()) {

diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp
@@ -74,6 +74,7 @@ class CellDbIn : public CellDbBase {
   CellDbIn(td::actor::ActorId<RootDb> root_db, td::actor::ActorId<CellDb> parent, std::string path,
            td::Ref<ValidatorManagerOptions> opts);
 
+  void validate_meta();
   void start_up() override;
   void alarm() override;