Merge branch 'ad-freiburg:master' into add-prefilter-is-datatype

.github/workflows/code-coverage.yml

@@ -51,6 +51,7 @@ jobs:
     - name: Install coverage tools
       run: |
         sudo apt install -y llvm-16
+        sudo apt install mold
     - name: Show path
       run: |
         which llvm-profdata-16
@@ -60,7 +61,7 @@ jobs:
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: cmake -B ${{github.workspace}}/build ${{env.cmake-flags}} -DCMAKE_BUILD_TYPE=${{env.build-type}} -DLOGLEVEL=TIMING -DADDITIONAL_COMPILER_FLAGS="${{env.warnings}} ${{env.asan-flags}} ${{env.ubsan-flags}} ${{env.coverage-flags}}" -DADDITIONAL_LINKER_FLAGS="${{env.coverage-flags}}" -DUSE_PARALLEL=false -DRUN_EXPENSIVE_TESTS=false -DSINGLE_TEST_BINARY=ON -DENABLE_EXPENSIVE_CHECKS=true
+      run: cmake -B ${{github.workspace}}/build ${{env.cmake-flags}} -DCMAKE_BUILD_TYPE=${{env.build-type}} -DLOGLEVEL=TIMING -DADDITIONAL_COMPILER_FLAGS="${{env.warnings}} ${{env.asan-flags}} ${{env.ubsan-flags}} ${{env.coverage-flags}}" -DADDITIONAL_LINKER_FLAGS="${{env.coverage-flags}}" -DUSE_PARALLEL=false -DRUN_EXPENSIVE_TESTS=false -DSINGLE_TEST_BINARY=ON -DENABLE_EXPENSIVE_CHECKS=true -DADDITIONAL_LINKER_FLAGS="-fuse-ld=mold"
 
     - name: Build
         # Build your program with the given configuration

.github/workflows/macos.yml

@@ -37,6 +37,9 @@ jobs:
         run: |
           pip3 install pyaml pyicu
 
+      - name: Install conan
+        run: |
+          brew install conan@2
       - name: Install dependencies
         run: |
           brew install llvm@16
@@ -46,27 +49,34 @@ jobs:
           echo 'export LDFLAGS="-L/usr/local/opt/llvm@16/lib -L/usr/local/opt/llvm@16/lib/c++ -Wl,-rpath,/usr/local/opt/llvm@16/lib/c++"' >> ~/.bash_profile
           echo LDFLAGS="-L/usr/local/opt/llvm@16/lib -L/usr/local/opt/llvm@16/lib/c++ -Wl,-rpath,/usr/local/opt/llvm@16/lib/c++" >> $GITHUB_ENV 
           echo 'export CPPFLAGS="-I/usr/local/opt/llvm@16/include"' >> ~/.bash_profile
-          echo  CPPFLAGS="/usr/local/opt/llvm@16/include" >> $GITHUB_ENV 
+          echo  CPPFLAGS="-I/usr/local/opt/llvm@16/include" >> $GITHUB_ENV
+          echo 'export CFLAGS="-I/usr/local/opt/llvm@16/include"' >> ~/.bash_profile
+          echo  CFLAGS="-I/usr/local/opt/llvm@16/include" >> $GITHUB_ENV
+          echo 'export CC="/usr/local/opt/llvm@16/bin/clang"' >> ~/.bash_profile
+          echo CC="/usr/local/opt/llvm@16/bin/clang" >> $GITHUB_ENV
+          echo 'export CXX="/usr/local/opt/llvm@16/bin/clang++"' >> ~/.bash_profile
+          echo CXX="/usr/local/opt/llvm@16/bin/clang++" >> $GITHUB_ENV
           source ~/.bash_profile
       - name: Print clang version
         run: clang++ --version
 
       - name: Cache for conan
         uses: actions/cache@v3
         env:
-          cache-name: cache-conan-modules
+          cache-name: cache-conan-modules-macos-13
         with:
           path: ~/.conan2
           key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('conanfile.txt') }}
       - name: Create build directory
         run: mkdir ${{github.workspace}}/build
       - name: Install and run conan
         working-directory: ${{github.workspace}}/build
-        run: >
-          conan install .. -pr:b=../conanprofiles/clang-16-macos -pr:h=../conanprofiles/clang-16-macos -of=. --build=missing;
+        run: |
+          conan install .. -pr:b=../conanprofiles/clang-16-macos -pr:h=../conanprofiles/clang-16-macos -of=. --build=missing
       - name: Configure CMake
         # For std::ranges::join_view we need the -fexperimental-library flag on libc++16, which on Mac requires to manually tinker with the linking flags.
-        run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -DCMAKE_TOOLCHAIN_FILE="$(pwd)/build/conan_toolchain.cmake" -DUSE_PARALLEL=true -DRUN_EXPENSIVE_TESTS=false -DENABLE_EXPENSIVE_CHECKS=true -DCMAKE_CXX_COMPILER=clang++ -DADDITIONAL_COMPILER_FLAGS="-fexperimental-library" -D_NO_TIMING_TESTS=ON -DADDITIONAL_LINKER_FLAGS="-L$(brew --prefix llvm)/lib/c++"
+        # We currently cannot use the parallel algorithms, as the parallel sort requires a GNU-extension, and we build with `libc++`.
+        run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -DCMAKE_TOOLCHAIN_FILE="$(pwd)/build/conan_toolchain.cmake" -DUSE_PARALLEL=false -DRUN_EXPENSIVE_TESTS=false -DENABLE_EXPENSIVE_CHECKS=true -DCMAKE_CXX_COMPILER=clang++ -DADDITIONAL_COMPILER_FLAGS="-fexperimental-library" -D_NO_TIMING_TESTS=ON -DADDITIONAL_LINKER_FLAGS="-L$(brew --prefix llvm)/lib/c++"
 
       - name: Build
         # Build your program with the given configuration

conanfile.txt

@@ -1,6 +1,6 @@
 [requires]
 boost/1.81.0
-icu/73.1
+icu/76.1
 openssl/3.1.1
 zstd/1.5.5
 # The jemalloc recipe for Conan2 is currently broken, uncomment this line as soon as this is fixed.

conanprofiles/clang-16-macos

@@ -6,3 +6,6 @@ compiler.cppstd=gnu17
 compiler.libcxx=libc++
 compiler.version=16
 os=Macos
+
+[conf]
+tools.build:compiler_executables={ "c": "clang", "cpp": "clang++"}

src/engine/Bind.cpp

@@ -220,3 +220,8 @@ IdTable Bind::computeExpressionBind(
 
   return idTable;
 }
+
+// _____________________________________________________________________________
+std::unique_ptr<Operation> Bind::cloneImpl() const {
+  return std::make_unique<Bind>(_executionContext, _subtree->clone(), _bind);
+}

src/engine/Bind.h

@@ -33,6 +33,7 @@ class Bind : public Operation {
   bool supportsLimit() const override;
 
  private:
+  std::unique_ptr<Operation> cloneImpl() const override;
   uint64_t getSizeEstimateBeforeLimit() override;
 
  public:

src/engine/CMakeLists.txt

@@ -15,5 +15,5 @@ add_library(engine
         TextLimit.cpp LazyGroupBy.cpp GroupByHashMapOptimization.cpp SpatialJoin.cpp
         CountConnectedSubgraphs.cpp SpatialJoinAlgorithms.cpp PathSearch.cpp ExecuteUpdate.cpp
         Describe.cpp GraphStoreProtocol.cpp
-        QueryExecutionContext.cpp ExistsJoin.cpp)
+        QueryExecutionContext.cpp ExistsJoin.cpp SPARQLProtocol.cpp ParsedRequestBuilder.cpp)
 qlever_target_link_libraries(engine util index parser sparqlExpressions http SortPerformanceEstimator Boost::iostreams s2)

src/engine/CartesianProductJoin.cpp

@@ -365,3 +365,14 @@ Result::Generator CartesianProductJoin::createLazyConsumer(
     idTables.pop_back();
   }
 }
+
+// _____________________________________________________________________________
+std::unique_ptr<Operation> CartesianProductJoin::cloneImpl() const {
+  Children copy;
+  copy.reserve(children_.size());
+  for (const auto& operation : children_) {
+    copy.push_back(operation->clone());
+  }
+  return std::make_unique<CartesianProductJoin>(_executionContext,
+                                                std::move(copy), chunkSize_);
+}

src/engine/CartesianProductJoin.h

@@ -61,6 +61,8 @@ class CartesianProductJoin : public Operation {
 
   VariableToColumnMap computeVariableToColumnMap() const override;
 
+  std::unique_ptr<Operation> cloneImpl() const override;
+
  public:
   float getMultiplicity([[maybe_unused]] size_t col) override;
 

src/engine/CountAvailablePredicates.cpp

@@ -360,3 +360,10 @@ void CountAvailablePredicates::computePatternTrick(
   runtimeInfo.addDetail("costRatio", costRatio * 100);
   *dynResult = std::move(result).toDynamic();
 }
+
+// _____________________________________________________________________________
+std::unique_ptr<Operation> CountAvailablePredicates::cloneImpl() const {
+  return std::make_unique<CountAvailablePredicates>(
+      _executionContext, subtree_->clone(), subjectColumnIndex_,
+      predicateVariable_, countVariable_);
+}

src/engine/CountAvailablePredicates.h

@@ -5,16 +5,11 @@
 
 #include <memory>
 #include <string>
-#include <utility>
 #include <vector>
 
-#include "../global/Pattern.h"
-#include "../parser/ParsedQuery.h"
-#include "./Operation.h"
-#include "./QueryExecutionTree.h"
-
-using std::string;
-using std::vector;
+#include "engine/Operation.h"
+#include "engine/QueryExecutionTree.h"
+#include "global/Pattern.h"
 
 // This Operation takes a Result with at least one column containing ids,
 // and a column index referring to such a column. It then creates a Result
@@ -43,16 +38,16 @@ class CountAvailablePredicates : public Operation {
                            Variable predicateVariable, Variable countVariable);
 
  protected:
-  [[nodiscard]] string getCacheKeyImpl() const override;
+  [[nodiscard]] std::string getCacheKeyImpl() const override;
 
  public:
-  [[nodiscard]] string getDescriptor() const override;
+  [[nodiscard]] std::string getDescriptor() const override;
 
   [[nodiscard]] size_t getResultWidth() const override;
 
-  [[nodiscard]] vector<ColumnIndex> resultSortedOn() const override;
+  [[nodiscard]] std::vector<ColumnIndex> resultSortedOn() const override;
 
-  vector<QueryExecutionTree*> getChildren() override {
+  std::vector<QueryExecutionTree*> getChildren() override {
     using R = vector<QueryExecutionTree*>;
     return subtree_ != nullptr ? R{subtree_.get()} : R{};
   }
@@ -69,6 +64,8 @@ class CountAvailablePredicates : public Operation {
  private:
   uint64_t getSizeEstimateBeforeLimit() override;
 
+  std::unique_ptr<Operation> cloneImpl() const override;
+
  public:
   size_t getCostEstimate() override;
 

src/engine/Describe.cpp

@@ -244,3 +244,9 @@ ProtoResult Describe::computeResult([[maybe_unused]] bool requestLaziness) {
 
   return {std::move(resultTable), resultSortedOn(), std::move(localVocab)};
 }
+
+// _____________________________________________________________________________
+std::unique_ptr<Operation> Describe::cloneImpl() const {
+  return std::make_unique<Describe>(_executionContext, subtree_->clone(),
+                                    describe_);
+}

src/engine/Describe.h

@@ -47,6 +47,7 @@ class Describe : public Operation {
   bool knownEmptyResult() override;
 
  private:
+  std::unique_ptr<Operation> cloneImpl() const override;
   [[nodiscard]] vector<ColumnIndex> resultSortedOn() const override;
   ProtoResult computeResult(bool requestLaziness) override;
   VariableToColumnMap computeVariableToColumnMap() const override;

src/engine/Distinct.cpp

@@ -182,3 +182,9 @@ IdTable Distinct::outOfPlaceDistinct(const IdTable& dynInput) const {
   LOG(DEBUG) << "Distinct done.\n";
   return std::move(output).toDynamic();
 }
+
+// _____________________________________________________________________________
+std::unique_ptr<Operation> Distinct::cloneImpl() const {
+  return std::make_unique<Distinct>(_executionContext, subtree_->clone(),
+                                    keepIndices_);
+}

src/engine/Distinct.h

@@ -52,6 +52,7 @@ class Distinct : public Operation {
   [[nodiscard]] string getCacheKeyImpl() const override;
 
  private:
+  std::unique_ptr<Operation> cloneImpl() const override;
   ProtoResult computeResult(bool requestLaziness) override;
 
   VariableToColumnMap computeVariableToColumnMap() const override;

src/engine/ExistsJoin.cpp

@@ -205,3 +205,11 @@ std::shared_ptr<QueryExecutionTree> ExistsJoin::addExistsJoinsToSubtree(
   }
   return subtree;
 }
+
+// _____________________________________________________________________________
+std::unique_ptr<Operation> ExistsJoin::cloneImpl() const {
+  auto newJoin = std::make_unique<ExistsJoin>(*this);
+  newJoin->left_ = left_->clone();
+  newJoin->right_ = right_->clone();
+  return newJoin;
+}

src/engine/ExistsJoin.h

@@ -76,6 +76,8 @@ class ExistsJoin : public Operation {
   }
 
  private:
+  std::unique_ptr<Operation> cloneImpl() const override;
+
   ProtoResult computeResult([[maybe_unused]] bool requestLaziness) override;
 
   VariableToColumnMap computeVariableToColumnMap() const override;

src/engine/ExportQueryExecutionTrees.cpp

@@ -14,6 +14,7 @@
 #include "parser/RdfEscaping.h"
 #include "util/ConstexprUtils.h"
 #include "util/http/MediaTypes.h"
+#include "util/json.h"
 
 // Return true iff the `result` is nonempty.
 bool getResultForAsk(const std::shared_ptr<const Result>& result) {

src/engine/ExportQueryExecutionTrees.h

@@ -10,7 +10,6 @@
 #include "parser/data/LimitOffsetClause.h"
 #include "util/CancellationHandle.h"
 #include "util/http/MediaTypes.h"
-#include "util/json.h"
 
 // Class for computing the result of an already parsed and planned query and
 // exporting it in different formats (TSV, CSV, Turtle, JSON, Binary).

src/engine/Filter.cpp

@@ -240,3 +240,9 @@ size_t Filter::getCostEstimate() {
                  _subtree->getRootOperation()->getPrimarySortKeyVariable())
              .costEstimate;
 }
+
+// _____________________________________________________________________________
+std::unique_ptr<Operation> Filter::cloneImpl() const {
+  return std::make_unique<Filter>(_executionContext, _subtree->clone(),
+                                  _expression);
+}

src/engine/Filter.h

@@ -10,7 +10,6 @@
 
 #include "engine/Operation.h"
 #include "engine/QueryExecutionTree.h"
-#include "parser/ParsedQuery.h"
 
 class Filter : public Operation {
   using PrefilterVariablePair = sparqlExpression::PrefilterExprVariablePair;
@@ -55,6 +54,8 @@ class Filter : public Operation {
   }
 
  private:
+  std::unique_ptr<Operation> cloneImpl() const override;
+
   VariableToColumnMap computeVariableToColumnMap() const override {
     return _subtree->getVariableColumns();
   }

src/engine/GraphStoreProtocol.cpp

@@ -4,29 +4,9 @@
 
 #include "engine/GraphStoreProtocol.h"
 
-#include "parser/Tokenizer.h"
+#include "parser/SparqlParser.h"
 #include "util/http/beast.h"
 
-// ____________________________________________________________________________
-GraphOrDefault GraphStoreProtocol::extractTargetGraph(
-    const ad_utility::url_parser::ParamValueMap& params) {
-  const std::optional<std::string> graphIri =
-      ad_utility::url_parser::checkParameter(params, "graph", std::nullopt);
-  const bool isDefault =
-      ad_utility::url_parser::checkParameter(params, "default", "").has_value();
-  if (graphIri.has_value() == isDefault) {
-    throw std::runtime_error(
-        "Exactly one of the query parameters default or graph must be set to "
-        "identify the graph for the graph store protocol request.");
-  }
-  if (graphIri.has_value()) {
-    return GraphRef::fromIrirefWithoutBrackets(graphIri.value());
-  } else {
-    AD_CORRECTNESS_CHECK(isDefault);
-    return DEFAULT{};
-  }
-}
-
 // ____________________________________________________________________________
 void GraphStoreProtocol::throwUnsupportedMediatype(
     const string_view& mediatype) {
@@ -85,18 +65,15 @@ std::vector<SparqlTripleSimpleWithGraph> GraphStoreProtocol::convertTriples(
 
 // ____________________________________________________________________________
 ParsedQuery GraphStoreProtocol::transformGet(const GraphOrDefault& graph) {
-  ParsedQuery res;
-  res._clause = parsedQuery::ConstructClause(
-      {{Variable("?s"), Variable("?p"), Variable("?o")}});
-  res._rootGraphPattern = {};
-  parsedQuery::GraphPattern selectSPO;
-  selectSPO._graphPatterns.emplace_back(parsedQuery::BasicGraphPattern{
-      {SparqlTriple(Variable("?s"), "?p", Variable("?o"))}});
+  // Construct the parsed query from its short equivalent SPARQL Update string.
+  // This is easier and also provides e.g. the `_originalString` field.
+  std::string query;
   if (const auto* iri =
           std::get_if<ad_utility::triple_component::Iri>(&graph)) {
-    res.datasetClauses_ =
-        parsedQuery::DatasetClauses::fromClauses({DatasetClause{*iri, false}});
+    query = absl::StrCat("CONSTRUCT { ?s ?p ?o } WHERE { GRAPH ",
+                         iri->toStringRepresentation(), " { ?s ?p ?o } }");
+  } else {
+    query = "CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }";
   }
-  res._rootGraphPattern = std::move(selectSPO);
-  return res;
+  return SparqlParser::parseQuery(query);
 }