From b182aeefb43352172d2a01deda9e7867e3740cc0 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Thu, 6 Feb 2025 19:05:48 +0000 Subject: [PATCH 1/3] FDB-389 removed TocCatalogue from TocIndex ctor --- src/fdb5/toc/TocCatalogueWriter.cc | 4 ++-- src/fdb5/toc/TocEngine.cc | 15 ++++++++------- src/fdb5/toc/TocEngine.h | 2 +- src/fdb5/toc/TocHandler.cc | 8 ++++---- src/fdb5/toc/TocIndex.cc | 2 -- src/fdb5/toc/TocIndex.h | 2 -- src/fdb5/tools/fdb-read.cc | 2 +- 7 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc index c9071afec..d9f5b9b48 100644 --- a/src/fdb5/toc/TocCatalogueWriter.cc +++ b/src/fdb5/toc/TocCatalogueWriter.cc @@ -66,7 +66,7 @@ bool TocCatalogueWriter::selectIndex(const Key& idxKey) { fdb5LustreapiFileCreate(indexPath, stripeIndexLustreSettings()); } - indexes_[idxKey] = Index(new TocIndex(idxKey, *this, indexPath, 0, TocIndex::WRITE)); + indexes_[idxKey] = Index(new TocIndex(idxKey, indexPath, 0, TocIndex::WRITE)); } current_ = indexes_[idxKey]; @@ -88,7 +88,7 @@ bool TocCatalogueWriter::selectIndex(const Key& idxKey) { fdb5LustreapiFileCreate(indexPath, stripeIndexLustreSettings()); } - fullIndexes_[idxKey] = Index(new TocIndex(idxKey, *this, indexPath, 0, TocIndex::WRITE)); + fullIndexes_[idxKey] = Index(new TocIndex(idxKey, indexPath, 0, TocIndex::WRITE)); } currentFull_ = fullIndexes_[idxKey]; diff --git a/src/fdb5/toc/TocEngine.cc b/src/fdb5/toc/TocEngine.cc index 0507621e0..313a33d40 100644 --- a/src/fdb5/toc/TocEngine.cc +++ b/src/fdb5/toc/TocEngine.cc @@ -42,7 +42,7 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -void TocEngine::scan_dbs(const std::string& path, std::list& dbs) const { +void TocEngine::scan_dbs(const std::string& path, std::list& dbs, bool lowercase) const { eckit::StdDir d(path.c_str()); @@ -73,7 +73,7 @@ void TocEngine::scan_dbs(const std::string& path, std::list& dbs) c #if defined(eckit_HAVE_DIRENT_D_TYPE) do_stat = false; if (e->d_type == DT_DIR) { - dbs.push_back(full); + dbs.push_back(lowercase ? eckit::StringTools::lower(full) : full); } else if (e->d_type == DT_UNKNOWN) { do_stat = true; } @@ -83,7 +83,7 @@ void TocEngine::scan_dbs(const std::string& path, std::list& dbs) c if(eckit::Stat::stat(full.c_str(), &info) == 0) { if(S_ISDIR(info.st_mode)) { - dbs.push_back(full); + dbs.push_back(lowercase ? eckit::StringTools::lower(full) : full); } } else Log::error() << "Cannot stat " << full << Log::syserr << std::endl; @@ -138,8 +138,9 @@ std::map TocEngine::databases(const std::mapasString())) + "/"; std::list dbs; - scan_dbs(*j, dbs); + scan_dbs(*j, dbs, true); for (const auto& [key, rule] : keys) { @@ -147,8 +148,8 @@ std::map TocEngine::databases(const std::mapasString()) + "/" + dbpath + "$"; - std::regex reg(regex, std::regex::icase | std::regex::optimize); + std::string regex = regex_prefix + eckit::StringTools::lower(dbpath) + "$"; + eckit::Regex reg(regex); LOG_DEBUG_LIB(LibFdb5) << " -> key " << key << " dbpath " << dbpath @@ -163,7 +164,7 @@ std::map TocEngine::databases(const std::map databases(const metkit::mars::MarsRequest& rq, const std::vector& dirs, const Config& config) const; - void scan_dbs(const std::string& path, std::list& dbs) const; + void scan_dbs(const std::string& path, std::list& dbs, bool lowercase = false) const; protected: // methods diff --git a/src/fdb5/toc/TocHandler.cc b/src/fdb5/toc/TocHandler.cc index 088aef479..20151c236 100644 --- a/src/fdb5/toc/TocHandler.cc +++ b/src/fdb5/toc/TocHandler.cc @@ -1340,7 +1340,7 @@ std::vector TocHandler::loadIndexes(const Catalogue& catalogue, bool sort s >> offset; s >> type; LOG_DEBUG(debug, LibFdb5) << "TocRecord TOC_INDEX " << path << " - " << offset << std::endl; - tocindexes[entry.seqNo] = new TocIndex(s, catalogue, entry.datap->header_.serialisationVersion_, + tocindexes[entry.seqNo] = new TocIndex(s, entry.datap->header_.serialisationVersion_, entry.tocDirectoryName, entry.tocDirectoryName / path, offset, preloadBTree_); @@ -1443,7 +1443,7 @@ void TocHandler::dump(std::ostream& out, bool simple, bool walkSubTocs) const { s >> type; out << " Path: " << path << ", offset: " << offset << ", type: " << type; if(!simple) { out << std::endl; } - Index index(new TocIndex(s, *(dynamic_cast(this)), r->header_.serialisationVersion_, + Index index(new TocIndex(s, r->header_.serialisationVersion_, currentDirectory(), currentDirectory() / path, offset)); index.dump(out, " ", simple); break; @@ -1500,7 +1500,7 @@ void TocHandler::dumpIndexFile(std::ostream& out, const eckit::PathName& indexFi if ((currentDirectory() / path).sameAs(eckit::LocalPathName{indexFile})) { r->dump(out, true); out << std::endl << " Path: " << path << ", offset: " << offset << ", type: " << type; - Index index(new TocIndex(s, *(dynamic_cast(this)), r->header_.serialisationVersion_, + Index index(new TocIndex(s, r->header_.serialisationVersion_, currentDirectory(), currentDirectory() / path, offset)); index.dump(out, " ", false, true); } @@ -1609,7 +1609,7 @@ void TocHandler::enumerateMasked(const Catalogue& catalogue, std::set key(absPath.baseName(), offset); if (maskedEntries_.find(key) != maskedEntries_.end()) { if (absPath.exists()) { - Index index(new TocIndex(s, *(dynamic_cast(this)), r->header_.serialisationVersion_, directory_, absPath, offset)); + Index index(new TocIndex(s, r->header_.serialisationVersion_, directory_, absPath, offset)); for (const auto& dataURI : index.dataURIs()) data.insert(dataURI); } } diff --git a/src/fdb5/toc/TocIndex.cc b/src/fdb5/toc/TocIndex.cc index 4b8228c1d..8cdbd4a14 100644 --- a/src/fdb5/toc/TocIndex.cc +++ b/src/fdb5/toc/TocIndex.cc @@ -46,7 +46,6 @@ class TocIndexCloser { /// the members of TocIndex TocIndex::TocIndex(const Key& key, - const Catalogue& catalogue, const eckit::PathName& path, off_t offset, Mode mode, @@ -60,7 +59,6 @@ TocIndex::TocIndex(const Key& key, preloadBTree_(false) { } TocIndex::TocIndex(eckit::Stream& s, - const Catalogue& catalogue, const int version, const eckit::PathName& directory, const eckit::PathName& path, diff --git a/src/fdb5/toc/TocIndex.h b/src/fdb5/toc/TocIndex.h index eede6dbd9..8f8ca9e0e 100644 --- a/src/fdb5/toc/TocIndex.h +++ b/src/fdb5/toc/TocIndex.h @@ -62,14 +62,12 @@ class TocIndex : public: // methods TocIndex(const Key& key, - const Catalogue& catalogue, const eckit::PathName &path, off_t offset, Mode mode, const std::string& type = defaulType()); TocIndex(eckit::Stream &, - const Catalogue& catalogue, const int version, const eckit::PathName &directory, const eckit::PathName &path, diff --git a/src/fdb5/tools/fdb-read.cc b/src/fdb5/tools/fdb-read.cc index eb6aaca4f..318ae8f43 100644 --- a/src/fdb5/tools/fdb-read.cc +++ b/src/fdb5/tools/fdb-read.cc @@ -98,7 +98,7 @@ void FDBRead::execute(const eckit::option::CmdArgs &args) { std::unique_ptr dh(handles.dataHandle()); - dh->copyTo(out); + dh->saveInto(out); } From 4e943ea913ebba336eaa94de03298a860aebda12 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 7 Feb 2025 10:50:42 +0100 Subject: [PATCH 2/3] chore(RootManager): tidy headers --- src/fdb5/toc/RootManager.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/fdb5/toc/RootManager.cc b/src/fdb5/toc/RootManager.cc index 3b3ae15fc..9841b8bd9 100644 --- a/src/fdb5/toc/RootManager.cc +++ b/src/fdb5/toc/RootManager.cc @@ -11,16 +11,17 @@ #include "RootManager.h" #include -#include +#include +#include +#include -#include "eckit/types/Types.h" #include "eckit/config/Resource.h" +#include "eckit/filesystem/LocalPathName.h" +#include "eckit/thread/AutoLock.h" +#include "eckit/thread/Mutex.h" +#include "eckit/types/Types.h" #include "eckit/utils/Tokenizer.h" -#include "eckit/utils/StringTools.h" #include "eckit/utils/Translator.h" -#include "eckit/thread/Mutex.h" -#include "eckit/thread/AutoLock.h" -#include "eckit/filesystem/LocalPathName.h" #include "metkit/mars/MarsRequest.h" From d9bf484567703e37d8cc8c10dc20419cf3d58154 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 7 Feb 2025 10:52:32 +0100 Subject: [PATCH 3/3] fix(RootManager): made key list unique --- src/fdb5/toc/RootManager.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/fdb5/toc/RootManager.cc b/src/fdb5/toc/RootManager.cc index 9841b8bd9..579660a54 100644 --- a/src/fdb5/toc/RootManager.cc +++ b/src/fdb5/toc/RootManager.cc @@ -631,9 +631,8 @@ std::vector RootManager::visitableRoots(const std::set& keys) { eckit::StringSet roots; - std::vector keystrings; - std::transform(keys.begin(), keys.end(), std::back_inserter(keystrings), - [](const Key& k) { return k.valuesToString(); }); + std::set keystrings; + for (const auto& key : keys) { keystrings.insert(key.valuesToString()); } LOG_DEBUG_LIB(LibFdb5) << "RootManager::visitableRoots() trying to match keys " << keystrings << std::endl;