Skip to content

Commit

Permalink
Give each index a unique ID
Browse files Browse the repository at this point in the history
The ID is a string that can be obtained via the API with
cmd=get-index-id. It is currently just a concatenation of the index name
and several statistics on the number of triples (which is reasonably
but not perfectly unique).

This feature is useful for applications like
https://github.com/ad-freiburg/qlever-petrimaps, which have an internal
cache that depends on the index. When the index changes, the application
should be able to notice this an clear its cache. With an index ID it
can.
  • Loading branch information
Hannah Bast committed Jan 18, 2024
1 parent d7635f0 commit d231e67
Show file tree
Hide file tree
Showing 6 changed files with 33 additions and 1 deletion.
4 changes: 4 additions & 0 deletions src/engine/Server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,10 @@ Awaitable<void> Server::process(
} else if (auto cmd = checkParameter("cmd", "get-settings")) {
logCommand(cmd, "get server settings");
response = createJsonResponse(RuntimeParameters().toMap(), request);
} else if (auto cmd = checkParameter("cmd", "get-index-id")) {
logCommand(cmd, "get index ID");
response = createOkResponse(index_.getIndexId(), request,
ad_utility::MediaType::textPlain);
} else if (auto cmd =
checkParameter("cmd", "dump-active-queries", accessTokenOk)) {
logCommand(cmd, "dump active queries");
Expand Down
3 changes: 3 additions & 0 deletions src/index/Index.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,9 @@ const std::string& Index::getTextName() const { return pimpl_->getTextName(); }
// ____________________________________________________________________________
const std::string& Index::getKbName() const { return pimpl_->getKbName(); }

// ____________________________________________________________________________
const std::string& Index::getIndexId() const { return pimpl_->getIndexId(); }

// ____________________________________________________________________________
Index::NumNormalAndInternal Index::numTriples() const {
return pimpl_->numTriples();
Expand Down
2 changes: 2 additions & 0 deletions src/index/Index.h
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,8 @@ class Index {

const std::string& getKbName() const;

const std::string& getIndexId() const;

NumNormalAndInternal numTriples() const;

size_t getNofTextRecords() const;
Expand Down
9 changes: 8 additions & 1 deletion src/index/IndexImpl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1025,12 +1025,19 @@ void IndexImpl::readConfiguration() {
};

loadDataMember("has-all-permutations", loadAllPermutations_, true);

loadDataMember("num-predicates-normal", numPredicatesNormal_);
// These might be missing if there are only two permutations.
loadDataMember("num-subjects-normal", numSubjectsNormal_, 0);
loadDataMember("num-objects-normal", numObjectsNormal_, 0);
loadDataMember("num-triples-normal", numTriplesNormal_);

// Compute unique ID for this index.
//
// TODO: This is a simplistic way. It would be better to incorporate bytes
// from the index files.
indexId_ = absl::StrCat("#", getKbName(), ".", numTriplesNormal_, ".",
numSubjectsNormal_, ".", numPredicatesNormal_, ".",
numObjectsNormal_);
}

// ___________________________________________________________________________
Expand Down
3 changes: 3 additions & 0 deletions src/index/IndexImpl.h
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ class IndexImpl {
size_t numPredicatesNormal_ = 0;
size_t numObjectsNormal_ = 0;
size_t numTriplesNormal_ = 0;
string indexId_;
/**
* @brief Maps pattern ids to sets of predicate ids.
*/
Expand Down Expand Up @@ -421,6 +422,8 @@ class IndexImpl {

const string& getKbName() const { return pso_.metaData().getName(); }

const string& getIndexId() const { return indexId_; }

size_t getNofTextRecords() const { return textMeta_.getNofTextRecords(); }
size_t getNofWordPostings() const { return textMeta_.getNofWordPostings(); }
size_t getNofEntityPostings() const {
Expand Down
13 changes: 13 additions & 0 deletions test/IndexTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,19 @@ TEST(IndexTest, createFromOnDiskIndexTest) {
ASSERT_TRUE(index.POS().metaData().getMetaData(b2).isFunctional());
};

TEST(IndexTest, indexId) {
std::string kb =
"<a1> <b> <c1> .\n"
"<a2> <b> <c2> .\n"
"<a2> <b> <c1> .\n"
"<a3> <b> <c2> .";
// Build index with all permutations (arg 2) and no patterns (arg 3). That
// way, we get four triples, two distinct subjects, one distinct predicate
// and two distinct objects.
const Index& index = getQec(kb, true, false)->getIndex();
ASSERT_EQ(index.getIndexId(), "#.4.3.1.2");
}

TEST(IndexTest, scanTest) {
auto testWithAndWithoutPrefixCompression = [](bool useCompression) {
using enum Permutation::Enum;
Expand Down

0 comments on commit d231e67

Please sign in to comment.