Merge pull request #36968 from vespa-engine/toregge/avoid-transient-memory-usage-overestimate-for-memory-index

Avoid transient memory usage overestimate for memory index.

Author: toregge

searchcore/src/tests/proton/index/indexmanager_test.cpp

@@ -438,8 +438,8 @@ TEST_F(IndexManagerTest, require_that_flush_stats_are_calculated) {
     search::memoryindex::DocumentInverterContext inverter_context(schema, *invertThreads, *pushThreads, fic);
     search::memoryindex::DocumentInverter        inverter(inverter_context);
 
-    uint64_t fixed_index_size = fic.getMemoryUsage().allocatedBytes();
-    uint64_t index_size = fic.getMemoryUsage().allocatedBytes() - fixed_index_size;
+    uint64_t fixed_index_size = fic.getMemoryUsage().usedBytes();
+    uint64_t index_size = fic.getMemoryUsage().usedBytes() - fixed_index_size;
     /// Must account for both docid 0 being reserved and the extra after.
     uint64_t selector_size = (1) * sizeof(Source);
     EXPECT_EQ(index_size, _index_manager->getMaintainer().getFlushStats().memory_before_bytes -
@@ -450,7 +450,7 @@ TEST_F(IndexManagerTest, require_that_flush_stats_are_calculated) {
     Document::UP doc = addDocument(docid);
     inverter.invertDocument(docid, *doc, {});
     push_documents_and_wait(inverter);
-    index_size = fic.getMemoryUsage().allocatedBytes() - fixed_index_size;
+    index_size = fic.getMemoryUsage().usedBytes() - fixed_index_size;
 
     /// Must account for both docid 0 being reserved and the extra after.
     selector_size = (docid + 1) * sizeof(Source);
@@ -462,10 +462,11 @@ TEST_F(IndexManagerTest, require_that_flush_stats_are_calculated) {
 
     doc = addDocument(docid + 10);
     inverter.invertDocument(docid + 10, *doc, {});
+    push_documents_and_wait(inverter);
     auto doc100 = addDocument(docid + 100);
     inverter.invertDocument(docid + 100, *doc100, {});
     push_documents_and_wait(inverter);
-    index_size = fic.getMemoryUsage().allocatedBytes() - fixed_index_size;
+    index_size = fic.getMemoryUsage().usedBytes() - fixed_index_size;
     /// Must account for both docid 0 being reserved and the extra after.
     selector_size = (docid + 100 + 1) * sizeof(Source);
     EXPECT_EQ(index_size, _index_manager->getMaintainer().getFlushStats().memory_before_bytes -

searchcore/src/vespa/searchcorespi/index/indexmaintainer.cpp

@@ -87,6 +87,10 @@ class ReconfigRunnableTask : public Executor::Task {
 
 ReconfigRunnableTask::~ReconfigRunnableTask() = default;
 
+uint64_t transient_memory_usage(uint64_t memory_used, uint64_t static_memory_footprint) {
+    return memory_used >= static_memory_footprint ? memory_used - static_memory_footprint : 0;
+}
+
 SerialNum noSerialNumHigh = std::numeric_limits<SerialNum>::max();
 
 } // namespace
@@ -956,11 +960,11 @@ IndexMaintainer::FlushStats IndexMaintainer::getFlushStats() const {
     {
         LockGuard lock(_index_update_lock);
         source_selector_bytes = _selector->getDocIdLimit() * sizeof(Source);
-        stats.memory_before_bytes += _current_index->getMemoryUsage().allocatedBytes() + source_selector_bytes;
+        stats.memory_before_bytes += _current_index->getMemoryUsage().usedBytes() + source_selector_bytes;
         stats.memory_after_bytes += _current_index->getStaticMemoryFootprint() + source_selector_bytes;
         numFrozen = _frozenMemoryIndexes.size();
         for (const FrozenMemoryIndexRef& frozen : _frozenMemoryIndexes) {
-            stats.memory_before_bytes += frozen._index->getMemoryUsage().allocatedBytes() + source_selector_bytes;
+            stats.memory_before_bytes += frozen._index->getMemoryUsage().usedBytes() + source_selector_bytes;
         }
         source_selector_changes = _source_selector_changes;
     }
@@ -1189,8 +1193,10 @@ search::IndexStats IndexMaintainer::get_index_stats(bool clear_disk_io_stats) co
 ResourceUsage IndexMaintainer::get_resource_usage() const {
     auto disk_indexes_resource_usage = _disk_indexes->get_resource_usage(*_layout);
     auto stats = get_index_stats(false);
+    auto memory_index_transient_memory_usage =
+        transient_memory_usage(stats.memoryUsage().usedBytes(), _current_index->getStaticMemoryFootprint());
     return ResourceUsage{
-        TransientResourceUsage{disk_indexes_resource_usage.transient().disk(), stats.memoryUsage().allocatedBytes()},
+        TransientResourceUsage{disk_indexes_resource_usage.transient().disk(), memory_index_transient_memory_usage},
         disk_indexes_resource_usage.disk()};
 }
 

searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp

@@ -434,14 +434,14 @@ TEST(MemoryIndexTest, require_that_we_understand_the_memory_footprint) {
     }
     {
         Index index(MySetup().field("f1"));
-        EXPECT_EQ(BASE_ALLOCATED, index.index.getStaticMemoryFootprint());
-        EXPECT_EQ(index.index.getStaticMemoryFootprint(), index.index.getMemoryUsage().allocatedBytes());
+        EXPECT_EQ(BASE_ALLOCATED, index.index.getMemoryUsage().allocatedBytes());
+        EXPECT_EQ(index.index.getStaticMemoryFootprint(), index.index.getMemoryUsage().usedBytes());
         EXPECT_EQ(BASE_USED, index.index.getMemoryUsage().usedBytes());
     }
     {
         Index index(MySetup().field("f1").field("f2"));
-        EXPECT_EQ(2 * BASE_ALLOCATED, index.index.getStaticMemoryFootprint());
-        EXPECT_EQ(index.index.getStaticMemoryFootprint(), index.index.getMemoryUsage().allocatedBytes());
+        EXPECT_EQ(2 * BASE_ALLOCATED, index.index.getMemoryUsage().allocatedBytes());
+        EXPECT_EQ(index.index.getStaticMemoryFootprint(), index.index.getMemoryUsage().usedBytes());
         EXPECT_EQ(2 * BASE_USED, index.index.getMemoryUsage().usedBytes());
         EXPECT_EQ(2 * BASE_USED, index.index.get_stats().memoryUsage().usedBytes());
         EXPECT_EQ(BASE_USED, get_field_stats(index.index.get_stats(), "f1").memory_usage().usedBytes());

searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp

@@ -73,7 +73,7 @@ MemoryIndex::MemoryIndex(const Schema& schema, const IFieldLengthInspector& insp
       _hiddenFields(schema.getNumIndexFields(), false),
       _prunedSchema(),
       _indexedDocs(0),
-      _staticMemoryFootprint(getMemoryUsage().allocatedBytes()) {
+      _staticMemoryFootprint(getMemoryUsage().usedBytes()) {
 }
 
 MemoryIndex::~MemoryIndex() = default;